From 8c0ab106c70f99ba44088af71d2d196125b15c1c Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 12 Dec 2020 13:32:06 +0100 Subject: [PATCH 001/522] initial commit --- Cargo.toml | 86 + README.md | 1 + build.rs | 10 + public/bulma.min.css | 1 + public/interface.html | 333 ++++ src/analytics.rs | 137 ++ src/data.rs | 162 ++ src/dump.rs | 425 +++++ src/error.rs | 307 ++++ src/helpers/authentication.rs | 103 ++ src/helpers/compression.rs | 27 + src/helpers/meilisearch.rs | 652 +++++++ src/helpers/mod.rs | 7 + src/helpers/normalize_path.rs | 86 + src/lib.rs | 104 ++ src/main.rs | 169 ++ src/models/mod.rs | 1 + src/models/update_operation.rs | 33 + src/option.rs | 221 +++ src/routes/document.rs | 266 +++ src/routes/dump.rs | 64 + src/routes/health.rs | 13 + src/routes/index.rs | 388 ++++ src/routes/key.rs | 26 + src/routes/mod.rs | 44 + src/routes/search.rs | 269 +++ src/routes/setting.rs | 547 ++++++ src/routes/stats.rs | 134 ++ src/routes/stop_words.rs | 79 + src/routes/synonym.rs | 90 + src/snapshot.rs | 96 + tests/assets/dumps/v1/metadata.json | 12 + tests/assets/dumps/v1/test/documents.jsonl | 77 + tests/assets/dumps/v1/test/settings.json | 59 + tests/assets/dumps/v1/test/updates.jsonl | 2 + tests/assets/test_set.json | 1613 +++++++++++++++++ tests/common.rs | 569 ++++++ tests/dashboard.rs | 12 + tests/documents_add.rs | 222 +++ tests/documents_delete.rs | 67 + tests/documents_get.rs | 23 + tests/dump.rs | 395 ++++ tests/errors.rs | 200 +++ tests/health.rs | 11 + tests/index.rs | 809 +++++++++ tests/index_update.rs | 200 +++ tests/lazy_index_creation.rs | 446 +++++ tests/placeholder_search.rs | 629 +++++++ tests/search.rs | 1879 ++++++++++++++++++++ tests/search_settings.rs | 538 ++++++ tests/settings.rs | 523 ++++++ tests/settings_ranking_rules.rs | 182 ++ tests/settings_stop_words.rs | 61 + tests/url_normalizer.rs | 18 + 54 files changed, 13428 insertions(+) create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 build.rs create mode 100644 public/bulma.min.css create mode 100644 public/interface.html create mode 100644 src/analytics.rs create mode 100644 src/data.rs create mode 100644 src/dump.rs create mode 100644 src/error.rs create mode 100644 src/helpers/authentication.rs create mode 100644 src/helpers/compression.rs create mode 100644 src/helpers/meilisearch.rs create mode 100644 src/helpers/mod.rs create mode 100644 src/helpers/normalize_path.rs create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/models/mod.rs create mode 100644 src/models/update_operation.rs create mode 100644 src/option.rs create mode 100644 src/routes/document.rs create mode 100644 src/routes/dump.rs create mode 100644 src/routes/health.rs create mode 100644 src/routes/index.rs create mode 100644 src/routes/key.rs create mode 100644 src/routes/mod.rs create mode 100644 src/routes/search.rs create mode 100644 src/routes/setting.rs create mode 100644 src/routes/stats.rs create mode 100644 src/routes/stop_words.rs create mode 100644 src/routes/synonym.rs create mode 100644 src/snapshot.rs create mode 100644 tests/assets/dumps/v1/metadata.json create mode 100644 tests/assets/dumps/v1/test/documents.jsonl create mode 100644 tests/assets/dumps/v1/test/settings.json create mode 100644 tests/assets/dumps/v1/test/updates.jsonl create mode 100644 tests/assets/test_set.json create mode 100644 tests/common.rs create mode 100644 tests/dashboard.rs create mode 100644 tests/documents_add.rs create mode 100644 tests/documents_delete.rs create mode 100644 tests/documents_get.rs create mode 100644 tests/dump.rs create mode 100644 tests/errors.rs create mode 100644 tests/health.rs create mode 100644 tests/index.rs create mode 100644 tests/index_update.rs create mode 100644 tests/lazy_index_creation.rs create mode 100644 tests/placeholder_search.rs create mode 100644 tests/search.rs create mode 100644 tests/search_settings.rs create mode 100644 tests/settings.rs create mode 100644 tests/settings_ranking_rules.rs create mode 100644 tests/settings_stop_words.rs create mode 100644 tests/url_normalizer.rs diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..6818443e6 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,86 @@ +[package] +name = "meilisearch-http" +description = "MeiliSearch HTTP server" +version = "0.17.0" +license = "MIT" +authors = [ + "Quentin de Quelen ", + "Clément Renault ", +] +edition = "2018" + +[[bin]] +name = "meilisearch" +path = "src/main.rs" + +[features] +default = ["sentry"] + +[dependencies] +actix-cors = "0.5.3" +actix-http = "2" +actix-rt = "1" +actix-service = "1.0.6" +actix-web = { version = "3.3.2", features = ["rustls"] } +bytes = "0.6.0" +chrono = { version = "0.4.19", features = ["serde"] } +crossbeam-channel = "0.5.0" +env_logger = "0.8.2" +flate2 = "1.0.18" +futures = "0.3.7" +http = "0.2.1" +indexmap = { version = "1.3.2", features = ["serde-1"] } +log = "0.4.8" +main_error = "0.1.0" +meilisearch-core = { path = "../meilisearch-core", version = "0.17.0" } +meilisearch-error = { path = "../meilisearch-error", version = "0.17.0" } +meilisearch-schema = { path = "../meilisearch-schema", version = "0.17.0" } +meilisearch-tokenizer = {path = "../meilisearch-tokenizer", version = "0.17.0"} +mime = "0.3.16" +once_cell = "1.5.2" +rand = "0.7.3" +regex = "1.4.2" +rustls = "0.18" +serde = { version = "1.0.117", features = ["derive"] } +serde_json = { version = "1.0.59", features = ["preserve_order"] } +serde_qs = "0.8.1" +sha2 = "0.9.1" +siphasher = "0.3.2" +slice-group-by = "0.2.6" +structopt = "0.3.20" +tar = "0.4.29" +tempfile = "3.1.0" +tokio = { version = "0.2.18", features = ["macros"] } +ureq = { version = "1.5.1", features = ["tls"], default-features = false } +walkdir = "2.3.1" +whoami = "1.0.0" + +[dependencies.sentry] +version = "0.18.1" +default-features = false +features = [ + "with_client_implementation", + "with_panic", + "with_failure", + "with_device_info", + "with_rust_info", + "with_reqwest_transport", + "with_rustls", + "with_env_logger" +] +optional = true + +[dev-dependencies] +serde_url_params = "0.2.0" +tempdir = "0.3.7" +tokio = { version = "0.2.18", features = ["macros", "time"] } + +[dev-dependencies.assert-json-diff] +git = "https://github.com/qdequele/assert-json-diff" +branch = "master" + +[build-dependencies] +vergen = "3.1.0" + +[target.'cfg(unix)'.dependencies] +jemallocator = "0.3.2" diff --git a/README.md b/README.md new file mode 100644 index 000000000..0271ef082 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# transplant of meilisearch http on the new engine diff --git a/build.rs b/build.rs new file mode 100644 index 000000000..2257407a8 --- /dev/null +++ b/build.rs @@ -0,0 +1,10 @@ +use vergen::{generate_cargo_keys, ConstantsFlags}; + +fn main() { + // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag + let mut flags = ConstantsFlags::all(); + flags.toggle(ConstantsFlags::SEMVER_FROM_CARGO_PKG); + + // Generate the 'cargo:' key output + generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!"); +} diff --git a/public/bulma.min.css b/public/bulma.min.css new file mode 100644 index 000000000..d0570ff03 --- /dev/null +++ b/public/bulma.min.css @@ -0,0 +1 @@ +/*! bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */@-webkit-keyframes spinAround{from{transform:rotate(0)}to{transform:rotate(359deg)}}@keyframes spinAround{from{transform:rotate(0)}to{transform:rotate(359deg)}}.breadcrumb,.button,.delete,.file,.is-unselectable,.modal-close,.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous,.tabs{-webkit-touch-callout:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.navbar-link:not(.is-arrowless)::after,.select:not(.is-multiple):not(.is-loading)::after{border:3px solid transparent;border-radius:2px;border-right:0;border-top:0;content:" ";display:block;height:.625em;margin-top:-.4375em;pointer-events:none;position:absolute;top:50%;transform:rotate(-45deg);transform-origin:center;width:.625em}.block:not(:last-child),.box:not(:last-child),.breadcrumb:not(:last-child),.content:not(:last-child),.highlight:not(:last-child),.level:not(:last-child),.list:not(:last-child),.message:not(:last-child),.notification:not(:last-child),.pagination:not(:last-child),.progress:not(:last-child),.subtitle:not(:last-child),.table-container:not(:last-child),.table:not(:last-child),.tabs:not(:last-child),.title:not(:last-child){margin-bottom:1.5rem}.delete,.modal-close{-moz-appearance:none;-webkit-appearance:none;background-color:rgba(10,10,10,.2);border:none;border-radius:290486px;cursor:pointer;pointer-events:auto;display:inline-block;flex-grow:0;flex-shrink:0;font-size:0;height:20px;max-height:20px;max-width:20px;min-height:20px;min-width:20px;outline:0;position:relative;vertical-align:top;width:20px}.delete::after,.delete::before,.modal-close::after,.modal-close::before{background-color:#fff;content:"";display:block;left:50%;position:absolute;top:50%;transform:translateX(-50%) translateY(-50%) rotate(45deg);transform-origin:center center}.delete::before,.modal-close::before{height:2px;width:50%}.delete::after,.modal-close::after{height:50%;width:2px}.delete:focus,.delete:hover,.modal-close:focus,.modal-close:hover{background-color:rgba(10,10,10,.3)}.delete:active,.modal-close:active{background-color:rgba(10,10,10,.4)}.is-small.delete,.is-small.modal-close{height:16px;max-height:16px;max-width:16px;min-height:16px;min-width:16px;width:16px}.is-medium.delete,.is-medium.modal-close{height:24px;max-height:24px;max-width:24px;min-height:24px;min-width:24px;width:24px}.is-large.delete,.is-large.modal-close{height:32px;max-height:32px;max-width:32px;min-height:32px;min-width:32px;width:32px}.button.is-loading::after,.control.is-loading::after,.loader,.select.is-loading::after{-webkit-animation:spinAround .5s infinite linear;animation:spinAround .5s infinite linear;border:2px solid #dbdbdb;border-radius:290486px;border-right-color:transparent;border-top-color:transparent;content:"";display:block;height:1em;position:relative;width:1em}.hero-video,.image.is-16by9 .has-ratio,.image.is-16by9 img,.image.is-1by1 .has-ratio,.image.is-1by1 img,.image.is-1by2 .has-ratio,.image.is-1by2 img,.image.is-1by3 .has-ratio,.image.is-1by3 img,.image.is-2by1 .has-ratio,.image.is-2by1 img,.image.is-2by3 .has-ratio,.image.is-2by3 img,.image.is-3by1 .has-ratio,.image.is-3by1 img,.image.is-3by2 .has-ratio,.image.is-3by2 img,.image.is-3by4 .has-ratio,.image.is-3by4 img,.image.is-3by5 .has-ratio,.image.is-3by5 img,.image.is-4by3 .has-ratio,.image.is-4by3 img,.image.is-4by5 .has-ratio,.image.is-4by5 img,.image.is-5by3 .has-ratio,.image.is-5by3 img,.image.is-5by4 .has-ratio,.image.is-5by4 img,.image.is-9by16 .has-ratio,.image.is-9by16 img,.image.is-square .has-ratio,.image.is-square img,.is-overlay,.modal,.modal-background{bottom:0;left:0;position:absolute;right:0;top:0}.button,.file-cta,.file-name,.input,.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous,.select select,.textarea{-moz-appearance:none;-webkit-appearance:none;align-items:center;border:1px solid transparent;border-radius:4px;box-shadow:none;display:inline-flex;font-size:1rem;height:2.5em;justify-content:flex-start;line-height:1.5;padding-bottom:calc(.5em - 1px);padding-left:calc(.75em - 1px);padding-right:calc(.75em - 1px);padding-top:calc(.5em - 1px);position:relative;vertical-align:top}.button:active,.button:focus,.file-cta:active,.file-cta:focus,.file-name:active,.file-name:focus,.input:active,.input:focus,.is-active.button,.is-active.file-cta,.is-active.file-name,.is-active.input,.is-active.pagination-ellipsis,.is-active.pagination-link,.is-active.pagination-next,.is-active.pagination-previous,.is-active.textarea,.is-focused.button,.is-focused.file-cta,.is-focused.file-name,.is-focused.input,.is-focused.pagination-ellipsis,.is-focused.pagination-link,.is-focused.pagination-next,.is-focused.pagination-previous,.is-focused.textarea,.pagination-ellipsis:active,.pagination-ellipsis:focus,.pagination-link:active,.pagination-link:focus,.pagination-next:active,.pagination-next:focus,.pagination-previous:active,.pagination-previous:focus,.select select.is-active,.select select.is-focused,.select select:active,.select select:focus,.textarea:active,.textarea:focus{outline:0}.button[disabled],.file-cta[disabled],.file-name[disabled],.input[disabled],.pagination-ellipsis[disabled],.pagination-link[disabled],.pagination-next[disabled],.pagination-previous[disabled],.select fieldset[disabled] select,.select select[disabled],.textarea[disabled],fieldset[disabled] .button,fieldset[disabled] .file-cta,fieldset[disabled] .file-name,fieldset[disabled] .input,fieldset[disabled] .pagination-ellipsis,fieldset[disabled] .pagination-link,fieldset[disabled] .pagination-next,fieldset[disabled] .pagination-previous,fieldset[disabled] .select select,fieldset[disabled] .textarea{cursor:not-allowed}/*! minireset.css v0.0.6 | MIT License | github.com/jgthms/minireset.css */blockquote,body,dd,dl,dt,fieldset,figure,h1,h2,h3,h4,h5,h6,hr,html,iframe,legend,li,ol,p,pre,textarea,ul{margin:0;padding:0}h1,h2,h3,h4,h5,h6{font-size:100%;font-weight:400}ul{list-style:none}button,input,select,textarea{margin:0}html{box-sizing:border-box}*,::after,::before{box-sizing:inherit}img,video{height:auto;max-width:100%}iframe{border:0}table{border-collapse:collapse;border-spacing:0}td,th{padding:0}td:not([align]),th:not([align]){text-align:left}html{background-color:#fff;font-size:16px;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;min-width:300px;overflow-x:hidden;overflow-y:scroll;text-rendering:optimizeLegibility;-webkit-text-size-adjust:100%;-moz-text-size-adjust:100%;-ms-text-size-adjust:100%;text-size-adjust:100%}article,aside,figure,footer,header,hgroup,section{display:block}body,button,input,select,textarea{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif}code,pre{-moz-osx-font-smoothing:auto;-webkit-font-smoothing:auto;font-family:monospace}body{color:#4a4a4a;font-size:1em;font-weight:400;line-height:1.5}a{color:#3273dc;cursor:pointer;text-decoration:none}a strong{color:currentColor}a:hover{color:#363636}code{background-color:#f5f5f5;color:#f14668;font-size:.875em;font-weight:400;padding:.25em .5em .25em}hr{background-color:#f5f5f5;border:none;display:block;height:2px;margin:1.5rem 0}img{height:auto;max-width:100%}input[type=checkbox],input[type=radio]{vertical-align:baseline}small{font-size:.875em}span{font-style:inherit;font-weight:inherit}strong{color:#363636;font-weight:700}fieldset{border:none}pre{-webkit-overflow-scrolling:touch;background-color:#f5f5f5;color:#4a4a4a;font-size:.875em;overflow-x:auto;padding:1.25rem 1.5rem;white-space:pre;word-wrap:normal}pre code{background-color:transparent;color:currentColor;font-size:1em;padding:0}table td,table th{vertical-align:top}table td:not([align]),table th:not([align]){text-align:left}table th{color:#363636}.is-clearfix::after{clear:both;content:" ";display:table}.is-pulled-left{float:left!important}.is-pulled-right{float:right!important}.is-clipped{overflow:hidden!important}.is-size-1{font-size:3rem!important}.is-size-2{font-size:2.5rem!important}.is-size-3{font-size:2rem!important}.is-size-4{font-size:1.5rem!important}.is-size-5{font-size:1.25rem!important}.is-size-6{font-size:1rem!important}.is-size-7{font-size:.75rem!important}@media screen and (max-width:768px){.is-size-1-mobile{font-size:3rem!important}.is-size-2-mobile{font-size:2.5rem!important}.is-size-3-mobile{font-size:2rem!important}.is-size-4-mobile{font-size:1.5rem!important}.is-size-5-mobile{font-size:1.25rem!important}.is-size-6-mobile{font-size:1rem!important}.is-size-7-mobile{font-size:.75rem!important}}@media screen and (min-width:769px),print{.is-size-1-tablet{font-size:3rem!important}.is-size-2-tablet{font-size:2.5rem!important}.is-size-3-tablet{font-size:2rem!important}.is-size-4-tablet{font-size:1.5rem!important}.is-size-5-tablet{font-size:1.25rem!important}.is-size-6-tablet{font-size:1rem!important}.is-size-7-tablet{font-size:.75rem!important}}@media screen and (max-width:1023px){.is-size-1-touch{font-size:3rem!important}.is-size-2-touch{font-size:2.5rem!important}.is-size-3-touch{font-size:2rem!important}.is-size-4-touch{font-size:1.5rem!important}.is-size-5-touch{font-size:1.25rem!important}.is-size-6-touch{font-size:1rem!important}.is-size-7-touch{font-size:.75rem!important}}@media screen and (min-width:1024px){.is-size-1-desktop{font-size:3rem!important}.is-size-2-desktop{font-size:2.5rem!important}.is-size-3-desktop{font-size:2rem!important}.is-size-4-desktop{font-size:1.5rem!important}.is-size-5-desktop{font-size:1.25rem!important}.is-size-6-desktop{font-size:1rem!important}.is-size-7-desktop{font-size:.75rem!important}}@media screen and (min-width:1216px){.is-size-1-widescreen{font-size:3rem!important}.is-size-2-widescreen{font-size:2.5rem!important}.is-size-3-widescreen{font-size:2rem!important}.is-size-4-widescreen{font-size:1.5rem!important}.is-size-5-widescreen{font-size:1.25rem!important}.is-size-6-widescreen{font-size:1rem!important}.is-size-7-widescreen{font-size:.75rem!important}}@media screen and (min-width:1408px){.is-size-1-fullhd{font-size:3rem!important}.is-size-2-fullhd{font-size:2.5rem!important}.is-size-3-fullhd{font-size:2rem!important}.is-size-4-fullhd{font-size:1.5rem!important}.is-size-5-fullhd{font-size:1.25rem!important}.is-size-6-fullhd{font-size:1rem!important}.is-size-7-fullhd{font-size:.75rem!important}}.has-text-centered{text-align:center!important}.has-text-justified{text-align:justify!important}.has-text-left{text-align:left!important}.has-text-right{text-align:right!important}@media screen and (max-width:768px){.has-text-centered-mobile{text-align:center!important}}@media screen and (min-width:769px),print{.has-text-centered-tablet{text-align:center!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-centered-tablet-only{text-align:center!important}}@media screen and (max-width:1023px){.has-text-centered-touch{text-align:center!important}}@media screen and (min-width:1024px){.has-text-centered-desktop{text-align:center!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-centered-desktop-only{text-align:center!important}}@media screen and (min-width:1216px){.has-text-centered-widescreen{text-align:center!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-centered-widescreen-only{text-align:center!important}}@media screen and (min-width:1408px){.has-text-centered-fullhd{text-align:center!important}}@media screen and (max-width:768px){.has-text-justified-mobile{text-align:justify!important}}@media screen and (min-width:769px),print{.has-text-justified-tablet{text-align:justify!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-justified-tablet-only{text-align:justify!important}}@media screen and (max-width:1023px){.has-text-justified-touch{text-align:justify!important}}@media screen and (min-width:1024px){.has-text-justified-desktop{text-align:justify!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-justified-desktop-only{text-align:justify!important}}@media screen and (min-width:1216px){.has-text-justified-widescreen{text-align:justify!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-justified-widescreen-only{text-align:justify!important}}@media screen and (min-width:1408px){.has-text-justified-fullhd{text-align:justify!important}}@media screen and (max-width:768px){.has-text-left-mobile{text-align:left!important}}@media screen and (min-width:769px),print{.has-text-left-tablet{text-align:left!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-left-tablet-only{text-align:left!important}}@media screen and (max-width:1023px){.has-text-left-touch{text-align:left!important}}@media screen and (min-width:1024px){.has-text-left-desktop{text-align:left!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-left-desktop-only{text-align:left!important}}@media screen and (min-width:1216px){.has-text-left-widescreen{text-align:left!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-left-widescreen-only{text-align:left!important}}@media screen and (min-width:1408px){.has-text-left-fullhd{text-align:left!important}}@media screen and (max-width:768px){.has-text-right-mobile{text-align:right!important}}@media screen and (min-width:769px),print{.has-text-right-tablet{text-align:right!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-right-tablet-only{text-align:right!important}}@media screen and (max-width:1023px){.has-text-right-touch{text-align:right!important}}@media screen and (min-width:1024px){.has-text-right-desktop{text-align:right!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-right-desktop-only{text-align:right!important}}@media screen and (min-width:1216px){.has-text-right-widescreen{text-align:right!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-right-widescreen-only{text-align:right!important}}@media screen and (min-width:1408px){.has-text-right-fullhd{text-align:right!important}}.is-capitalized{text-transform:capitalize!important}.is-lowercase{text-transform:lowercase!important}.is-uppercase{text-transform:uppercase!important}.is-italic{font-style:italic!important}.has-text-white{color:#fff!important}a.has-text-white:focus,a.has-text-white:hover{color:#e6e6e6!important}.has-background-white{background-color:#fff!important}.has-text-black{color:#0a0a0a!important}a.has-text-black:focus,a.has-text-black:hover{color:#000!important}.has-background-black{background-color:#0a0a0a!important}.has-text-light{color:#f5f5f5!important}a.has-text-light:focus,a.has-text-light:hover{color:#dbdbdb!important}.has-background-light{background-color:#f5f5f5!important}.has-text-dark{color:#363636!important}a.has-text-dark:focus,a.has-text-dark:hover{color:#1c1c1c!important}.has-background-dark{background-color:#363636!important}.has-text-primary{color:#00d1b2!important}a.has-text-primary:focus,a.has-text-primary:hover{color:#009e86!important}.has-background-primary{background-color:#00d1b2!important}.has-text-link{color:#3273dc!important}a.has-text-link:focus,a.has-text-link:hover{color:#205bbc!important}.has-background-link{background-color:#3273dc!important}.has-text-info{color:#3298dc!important}a.has-text-info:focus,a.has-text-info:hover{color:#207dbc!important}.has-background-info{background-color:#3298dc!important}.has-text-success{color:#48c774!important}a.has-text-success:focus,a.has-text-success:hover{color:#34a85c!important}.has-background-success{background-color:#48c774!important}.has-text-warning{color:#ffdd57!important}a.has-text-warning:focus,a.has-text-warning:hover{color:#ffd324!important}.has-background-warning{background-color:#ffdd57!important}.has-text-danger{color:#f14668!important}a.has-text-danger:focus,a.has-text-danger:hover{color:#ee1742!important}.has-background-danger{background-color:#f14668!important}.has-text-black-bis{color:#121212!important}.has-background-black-bis{background-color:#121212!important}.has-text-black-ter{color:#242424!important}.has-background-black-ter{background-color:#242424!important}.has-text-grey-darker{color:#363636!important}.has-background-grey-darker{background-color:#363636!important}.has-text-grey-dark{color:#4a4a4a!important}.has-background-grey-dark{background-color:#4a4a4a!important}.has-text-grey{color:#7a7a7a!important}.has-background-grey{background-color:#7a7a7a!important}.has-text-grey-light{color:#b5b5b5!important}.has-background-grey-light{background-color:#b5b5b5!important}.has-text-grey-lighter{color:#dbdbdb!important}.has-background-grey-lighter{background-color:#dbdbdb!important}.has-text-white-ter{color:#f5f5f5!important}.has-background-white-ter{background-color:#f5f5f5!important}.has-text-white-bis{color:#fafafa!important}.has-background-white-bis{background-color:#fafafa!important}.has-text-weight-light{font-weight:300!important}.has-text-weight-normal{font-weight:400!important}.has-text-weight-medium{font-weight:500!important}.has-text-weight-semibold{font-weight:600!important}.has-text-weight-bold{font-weight:700!important}.is-family-primary{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-secondary{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-sans-serif{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-monospace{font-family:monospace!important}.is-family-code{font-family:monospace!important}.is-block{display:block!important}@media screen and (max-width:768px){.is-block-mobile{display:block!important}}@media screen and (min-width:769px),print{.is-block-tablet{display:block!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-block-tablet-only{display:block!important}}@media screen and (max-width:1023px){.is-block-touch{display:block!important}}@media screen and (min-width:1024px){.is-block-desktop{display:block!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-block-desktop-only{display:block!important}}@media screen and (min-width:1216px){.is-block-widescreen{display:block!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-block-widescreen-only{display:block!important}}@media screen and (min-width:1408px){.is-block-fullhd{display:block!important}}.is-flex{display:flex!important}@media screen and (max-width:768px){.is-flex-mobile{display:flex!important}}@media screen and (min-width:769px),print{.is-flex-tablet{display:flex!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-flex-tablet-only{display:flex!important}}@media screen and (max-width:1023px){.is-flex-touch{display:flex!important}}@media screen and (min-width:1024px){.is-flex-desktop{display:flex!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-flex-desktop-only{display:flex!important}}@media screen and (min-width:1216px){.is-flex-widescreen{display:flex!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-flex-widescreen-only{display:flex!important}}@media screen and (min-width:1408px){.is-flex-fullhd{display:flex!important}}.is-inline{display:inline!important}@media screen and (max-width:768px){.is-inline-mobile{display:inline!important}}@media screen and (min-width:769px),print{.is-inline-tablet{display:inline!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-tablet-only{display:inline!important}}@media screen and (max-width:1023px){.is-inline-touch{display:inline!important}}@media screen and (min-width:1024px){.is-inline-desktop{display:inline!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-desktop-only{display:inline!important}}@media screen and (min-width:1216px){.is-inline-widescreen{display:inline!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-widescreen-only{display:inline!important}}@media screen and (min-width:1408px){.is-inline-fullhd{display:inline!important}}.is-inline-block{display:inline-block!important}@media screen and (max-width:768px){.is-inline-block-mobile{display:inline-block!important}}@media screen and (min-width:769px),print{.is-inline-block-tablet{display:inline-block!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-block-tablet-only{display:inline-block!important}}@media screen and (max-width:1023px){.is-inline-block-touch{display:inline-block!important}}@media screen and (min-width:1024px){.is-inline-block-desktop{display:inline-block!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-block-desktop-only{display:inline-block!important}}@media screen and (min-width:1216px){.is-inline-block-widescreen{display:inline-block!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-block-widescreen-only{display:inline-block!important}}@media screen and (min-width:1408px){.is-inline-block-fullhd{display:inline-block!important}}.is-inline-flex{display:inline-flex!important}@media screen and (max-width:768px){.is-inline-flex-mobile{display:inline-flex!important}}@media screen and (min-width:769px),print{.is-inline-flex-tablet{display:inline-flex!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-flex-tablet-only{display:inline-flex!important}}@media screen and (max-width:1023px){.is-inline-flex-touch{display:inline-flex!important}}@media screen and (min-width:1024px){.is-inline-flex-desktop{display:inline-flex!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-flex-desktop-only{display:inline-flex!important}}@media screen and (min-width:1216px){.is-inline-flex-widescreen{display:inline-flex!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-flex-widescreen-only{display:inline-flex!important}}@media screen and (min-width:1408px){.is-inline-flex-fullhd{display:inline-flex!important}}.is-hidden{display:none!important}.is-sr-only{border:none!important;clip:rect(0,0,0,0)!important;height:.01em!important;overflow:hidden!important;padding:0!important;position:absolute!important;white-space:nowrap!important;width:.01em!important}@media screen and (max-width:768px){.is-hidden-mobile{display:none!important}}@media screen and (min-width:769px),print{.is-hidden-tablet{display:none!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-hidden-tablet-only{display:none!important}}@media screen and (max-width:1023px){.is-hidden-touch{display:none!important}}@media screen and (min-width:1024px){.is-hidden-desktop{display:none!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-hidden-desktop-only{display:none!important}}@media screen and (min-width:1216px){.is-hidden-widescreen{display:none!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-hidden-widescreen-only{display:none!important}}@media screen and (min-width:1408px){.is-hidden-fullhd{display:none!important}}.is-invisible{visibility:hidden!important}@media screen and (max-width:768px){.is-invisible-mobile{visibility:hidden!important}}@media screen and (min-width:769px),print{.is-invisible-tablet{visibility:hidden!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-invisible-tablet-only{visibility:hidden!important}}@media screen and (max-width:1023px){.is-invisible-touch{visibility:hidden!important}}@media screen and (min-width:1024px){.is-invisible-desktop{visibility:hidden!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-invisible-desktop-only{visibility:hidden!important}}@media screen and (min-width:1216px){.is-invisible-widescreen{visibility:hidden!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-invisible-widescreen-only{visibility:hidden!important}}@media screen and (min-width:1408px){.is-invisible-fullhd{visibility:hidden!important}}.is-marginless{margin:0!important}.is-paddingless{padding:0!important}.is-radiusless{border-radius:0!important}.is-shadowless{box-shadow:none!important}.is-relative{position:relative!important}.box{background-color:#fff;border-radius:6px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);color:#4a4a4a;display:block;padding:1.25rem}a.box:focus,a.box:hover{box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px #3273dc}a.box:active{box-shadow:inset 0 1px 2px rgba(10,10,10,.2),0 0 0 1px #3273dc}.button{background-color:#fff;border-color:#dbdbdb;border-width:1px;color:#363636;cursor:pointer;justify-content:center;padding-bottom:calc(.5em - 1px);padding-left:1em;padding-right:1em;padding-top:calc(.5em - 1px);text-align:center;white-space:nowrap}.button strong{color:inherit}.button .icon,.button .icon.is-large,.button .icon.is-medium,.button .icon.is-small{height:1.5em;width:1.5em}.button .icon:first-child:not(:last-child){margin-left:calc(-.5em - 1px);margin-right:.25em}.button .icon:last-child:not(:first-child){margin-left:.25em;margin-right:calc(-.5em - 1px)}.button .icon:first-child:last-child{margin-left:calc(-.5em - 1px);margin-right:calc(-.5em - 1px)}.button.is-hovered,.button:hover{border-color:#b5b5b5;color:#363636}.button.is-focused,.button:focus{border-color:#3273dc;color:#363636}.button.is-focused:not(:active),.button:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.button.is-active,.button:active{border-color:#4a4a4a;color:#363636}.button.is-text{background-color:transparent;border-color:transparent;color:#4a4a4a;text-decoration:underline}.button.is-text.is-focused,.button.is-text.is-hovered,.button.is-text:focus,.button.is-text:hover{background-color:#f5f5f5;color:#363636}.button.is-text.is-active,.button.is-text:active{background-color:#e8e8e8;color:#363636}.button.is-text[disabled],fieldset[disabled] .button.is-text{background-color:transparent;border-color:transparent;box-shadow:none}.button.is-white{background-color:#fff;border-color:transparent;color:#0a0a0a}.button.is-white.is-hovered,.button.is-white:hover{background-color:#f9f9f9;border-color:transparent;color:#0a0a0a}.button.is-white.is-focused,.button.is-white:focus{border-color:transparent;color:#0a0a0a}.button.is-white.is-focused:not(:active),.button.is-white:focus:not(:active){box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.button.is-white.is-active,.button.is-white:active{background-color:#f2f2f2;border-color:transparent;color:#0a0a0a}.button.is-white[disabled],fieldset[disabled] .button.is-white{background-color:#fff;border-color:transparent;box-shadow:none}.button.is-white.is-inverted{background-color:#0a0a0a;color:#fff}.button.is-white.is-inverted.is-hovered,.button.is-white.is-inverted:hover{background-color:#000}.button.is-white.is-inverted[disabled],fieldset[disabled] .button.is-white.is-inverted{background-color:#0a0a0a;border-color:transparent;box-shadow:none;color:#fff}.button.is-white.is-loading::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-white.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-white.is-outlined.is-focused,.button.is-white.is-outlined.is-hovered,.button.is-white.is-outlined:focus,.button.is-white.is-outlined:hover{background-color:#fff;border-color:#fff;color:#0a0a0a}.button.is-white.is-outlined.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-white.is-outlined.is-loading.is-focused::after,.button.is-white.is-outlined.is-loading.is-hovered::after,.button.is-white.is-outlined.is-loading:focus::after,.button.is-white.is-outlined.is-loading:hover::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-white.is-outlined[disabled],fieldset[disabled] .button.is-white.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-white.is-inverted.is-outlined{background-color:transparent;border-color:#0a0a0a;color:#0a0a0a}.button.is-white.is-inverted.is-outlined.is-focused,.button.is-white.is-inverted.is-outlined.is-hovered,.button.is-white.is-inverted.is-outlined:focus,.button.is-white.is-inverted.is-outlined:hover{background-color:#0a0a0a;color:#fff}.button.is-white.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-white.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-white.is-inverted.is-outlined.is-loading:focus::after,.button.is-white.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-white.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-white.is-inverted.is-outlined{background-color:transparent;border-color:#0a0a0a;box-shadow:none;color:#0a0a0a}.button.is-black{background-color:#0a0a0a;border-color:transparent;color:#fff}.button.is-black.is-hovered,.button.is-black:hover{background-color:#040404;border-color:transparent;color:#fff}.button.is-black.is-focused,.button.is-black:focus{border-color:transparent;color:#fff}.button.is-black.is-focused:not(:active),.button.is-black:focus:not(:active){box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.button.is-black.is-active,.button.is-black:active{background-color:#000;border-color:transparent;color:#fff}.button.is-black[disabled],fieldset[disabled] .button.is-black{background-color:#0a0a0a;border-color:transparent;box-shadow:none}.button.is-black.is-inverted{background-color:#fff;color:#0a0a0a}.button.is-black.is-inverted.is-hovered,.button.is-black.is-inverted:hover{background-color:#f2f2f2}.button.is-black.is-inverted[disabled],fieldset[disabled] .button.is-black.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#0a0a0a}.button.is-black.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-black.is-outlined{background-color:transparent;border-color:#0a0a0a;color:#0a0a0a}.button.is-black.is-outlined.is-focused,.button.is-black.is-outlined.is-hovered,.button.is-black.is-outlined:focus,.button.is-black.is-outlined:hover{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.button.is-black.is-outlined.is-loading::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-black.is-outlined.is-loading.is-focused::after,.button.is-black.is-outlined.is-loading.is-hovered::after,.button.is-black.is-outlined.is-loading:focus::after,.button.is-black.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-black.is-outlined[disabled],fieldset[disabled] .button.is-black.is-outlined{background-color:transparent;border-color:#0a0a0a;box-shadow:none;color:#0a0a0a}.button.is-black.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-black.is-inverted.is-outlined.is-focused,.button.is-black.is-inverted.is-outlined.is-hovered,.button.is-black.is-inverted.is-outlined:focus,.button.is-black.is-inverted.is-outlined:hover{background-color:#fff;color:#0a0a0a}.button.is-black.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-black.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-black.is-inverted.is-outlined.is-loading:focus::after,.button.is-black.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-black.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-black.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-light{background-color:#f5f5f5;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-hovered,.button.is-light:hover{background-color:#eee;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-focused,.button.is-light:focus{border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-focused:not(:active),.button.is-light:focus:not(:active){box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.button.is-light.is-active,.button.is-light:active{background-color:#e8e8e8;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light[disabled],fieldset[disabled] .button.is-light{background-color:#f5f5f5;border-color:transparent;box-shadow:none}.button.is-light.is-inverted{background-color:rgba(0,0,0,.7);color:#f5f5f5}.button.is-light.is-inverted.is-hovered,.button.is-light.is-inverted:hover{background-color:rgba(0,0,0,.7)}.button.is-light.is-inverted[disabled],fieldset[disabled] .button.is-light.is-inverted{background-color:rgba(0,0,0,.7);border-color:transparent;box-shadow:none;color:#f5f5f5}.button.is-light.is-loading::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-light.is-outlined{background-color:transparent;border-color:#f5f5f5;color:#f5f5f5}.button.is-light.is-outlined.is-focused,.button.is-light.is-outlined.is-hovered,.button.is-light.is-outlined:focus,.button.is-light.is-outlined:hover{background-color:#f5f5f5;border-color:#f5f5f5;color:rgba(0,0,0,.7)}.button.is-light.is-outlined.is-loading::after{border-color:transparent transparent #f5f5f5 #f5f5f5!important}.button.is-light.is-outlined.is-loading.is-focused::after,.button.is-light.is-outlined.is-loading.is-hovered::after,.button.is-light.is-outlined.is-loading:focus::after,.button.is-light.is-outlined.is-loading:hover::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-light.is-outlined[disabled],fieldset[disabled] .button.is-light.is-outlined{background-color:transparent;border-color:#f5f5f5;box-shadow:none;color:#f5f5f5}.button.is-light.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);color:rgba(0,0,0,.7)}.button.is-light.is-inverted.is-outlined.is-focused,.button.is-light.is-inverted.is-outlined.is-hovered,.button.is-light.is-inverted.is-outlined:focus,.button.is-light.is-inverted.is-outlined:hover{background-color:rgba(0,0,0,.7);color:#f5f5f5}.button.is-light.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-light.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-light.is-inverted.is-outlined.is-loading:focus::after,.button.is-light.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #f5f5f5 #f5f5f5!important}.button.is-light.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-light.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);box-shadow:none;color:rgba(0,0,0,.7)}.button.is-dark{background-color:#363636;border-color:transparent;color:#fff}.button.is-dark.is-hovered,.button.is-dark:hover{background-color:#2f2f2f;border-color:transparent;color:#fff}.button.is-dark.is-focused,.button.is-dark:focus{border-color:transparent;color:#fff}.button.is-dark.is-focused:not(:active),.button.is-dark:focus:not(:active){box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.button.is-dark.is-active,.button.is-dark:active{background-color:#292929;border-color:transparent;color:#fff}.button.is-dark[disabled],fieldset[disabled] .button.is-dark{background-color:#363636;border-color:transparent;box-shadow:none}.button.is-dark.is-inverted{background-color:#fff;color:#363636}.button.is-dark.is-inverted.is-hovered,.button.is-dark.is-inverted:hover{background-color:#f2f2f2}.button.is-dark.is-inverted[disabled],fieldset[disabled] .button.is-dark.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#363636}.button.is-dark.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-dark.is-outlined{background-color:transparent;border-color:#363636;color:#363636}.button.is-dark.is-outlined.is-focused,.button.is-dark.is-outlined.is-hovered,.button.is-dark.is-outlined:focus,.button.is-dark.is-outlined:hover{background-color:#363636;border-color:#363636;color:#fff}.button.is-dark.is-outlined.is-loading::after{border-color:transparent transparent #363636 #363636!important}.button.is-dark.is-outlined.is-loading.is-focused::after,.button.is-dark.is-outlined.is-loading.is-hovered::after,.button.is-dark.is-outlined.is-loading:focus::after,.button.is-dark.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-dark.is-outlined[disabled],fieldset[disabled] .button.is-dark.is-outlined{background-color:transparent;border-color:#363636;box-shadow:none;color:#363636}.button.is-dark.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-dark.is-inverted.is-outlined.is-focused,.button.is-dark.is-inverted.is-outlined.is-hovered,.button.is-dark.is-inverted.is-outlined:focus,.button.is-dark.is-inverted.is-outlined:hover{background-color:#fff;color:#363636}.button.is-dark.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-dark.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-dark.is-inverted.is-outlined.is-loading:focus::after,.button.is-dark.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #363636 #363636!important}.button.is-dark.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-dark.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-primary{background-color:#00d1b2;border-color:transparent;color:#fff}.button.is-primary.is-hovered,.button.is-primary:hover{background-color:#00c4a7;border-color:transparent;color:#fff}.button.is-primary.is-focused,.button.is-primary:focus{border-color:transparent;color:#fff}.button.is-primary.is-focused:not(:active),.button.is-primary:focus:not(:active){box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.button.is-primary.is-active,.button.is-primary:active{background-color:#00b89c;border-color:transparent;color:#fff}.button.is-primary[disabled],fieldset[disabled] .button.is-primary{background-color:#00d1b2;border-color:transparent;box-shadow:none}.button.is-primary.is-inverted{background-color:#fff;color:#00d1b2}.button.is-primary.is-inverted.is-hovered,.button.is-primary.is-inverted:hover{background-color:#f2f2f2}.button.is-primary.is-inverted[disabled],fieldset[disabled] .button.is-primary.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#00d1b2}.button.is-primary.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-primary.is-outlined{background-color:transparent;border-color:#00d1b2;color:#00d1b2}.button.is-primary.is-outlined.is-focused,.button.is-primary.is-outlined.is-hovered,.button.is-primary.is-outlined:focus,.button.is-primary.is-outlined:hover{background-color:#00d1b2;border-color:#00d1b2;color:#fff}.button.is-primary.is-outlined.is-loading::after{border-color:transparent transparent #00d1b2 #00d1b2!important}.button.is-primary.is-outlined.is-loading.is-focused::after,.button.is-primary.is-outlined.is-loading.is-hovered::after,.button.is-primary.is-outlined.is-loading:focus::after,.button.is-primary.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-primary.is-outlined[disabled],fieldset[disabled] .button.is-primary.is-outlined{background-color:transparent;border-color:#00d1b2;box-shadow:none;color:#00d1b2}.button.is-primary.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-primary.is-inverted.is-outlined.is-focused,.button.is-primary.is-inverted.is-outlined.is-hovered,.button.is-primary.is-inverted.is-outlined:focus,.button.is-primary.is-inverted.is-outlined:hover{background-color:#fff;color:#00d1b2}.button.is-primary.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-primary.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-primary.is-inverted.is-outlined.is-loading:focus::after,.button.is-primary.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #00d1b2 #00d1b2!important}.button.is-primary.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-primary.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-primary.is-light{background-color:#ebfffc;color:#00947e}.button.is-primary.is-light.is-hovered,.button.is-primary.is-light:hover{background-color:#defffa;border-color:transparent;color:#00947e}.button.is-primary.is-light.is-active,.button.is-primary.is-light:active{background-color:#d1fff8;border-color:transparent;color:#00947e}.button.is-link{background-color:#3273dc;border-color:transparent;color:#fff}.button.is-link.is-hovered,.button.is-link:hover{background-color:#276cda;border-color:transparent;color:#fff}.button.is-link.is-focused,.button.is-link:focus{border-color:transparent;color:#fff}.button.is-link.is-focused:not(:active),.button.is-link:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.button.is-link.is-active,.button.is-link:active{background-color:#2366d1;border-color:transparent;color:#fff}.button.is-link[disabled],fieldset[disabled] .button.is-link{background-color:#3273dc;border-color:transparent;box-shadow:none}.button.is-link.is-inverted{background-color:#fff;color:#3273dc}.button.is-link.is-inverted.is-hovered,.button.is-link.is-inverted:hover{background-color:#f2f2f2}.button.is-link.is-inverted[disabled],fieldset[disabled] .button.is-link.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#3273dc}.button.is-link.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-link.is-outlined{background-color:transparent;border-color:#3273dc;color:#3273dc}.button.is-link.is-outlined.is-focused,.button.is-link.is-outlined.is-hovered,.button.is-link.is-outlined:focus,.button.is-link.is-outlined:hover{background-color:#3273dc;border-color:#3273dc;color:#fff}.button.is-link.is-outlined.is-loading::after{border-color:transparent transparent #3273dc #3273dc!important}.button.is-link.is-outlined.is-loading.is-focused::after,.button.is-link.is-outlined.is-loading.is-hovered::after,.button.is-link.is-outlined.is-loading:focus::after,.button.is-link.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-link.is-outlined[disabled],fieldset[disabled] .button.is-link.is-outlined{background-color:transparent;border-color:#3273dc;box-shadow:none;color:#3273dc}.button.is-link.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-link.is-inverted.is-outlined.is-focused,.button.is-link.is-inverted.is-outlined.is-hovered,.button.is-link.is-inverted.is-outlined:focus,.button.is-link.is-inverted.is-outlined:hover{background-color:#fff;color:#3273dc}.button.is-link.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-link.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-link.is-inverted.is-outlined.is-loading:focus::after,.button.is-link.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #3273dc #3273dc!important}.button.is-link.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-link.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-link.is-light{background-color:#eef3fc;color:#2160c4}.button.is-link.is-light.is-hovered,.button.is-link.is-light:hover{background-color:#e3ecfa;border-color:transparent;color:#2160c4}.button.is-link.is-light.is-active,.button.is-link.is-light:active{background-color:#d8e4f8;border-color:transparent;color:#2160c4}.button.is-info{background-color:#3298dc;border-color:transparent;color:#fff}.button.is-info.is-hovered,.button.is-info:hover{background-color:#2793da;border-color:transparent;color:#fff}.button.is-info.is-focused,.button.is-info:focus{border-color:transparent;color:#fff}.button.is-info.is-focused:not(:active),.button.is-info:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.button.is-info.is-active,.button.is-info:active{background-color:#238cd1;border-color:transparent;color:#fff}.button.is-info[disabled],fieldset[disabled] .button.is-info{background-color:#3298dc;border-color:transparent;box-shadow:none}.button.is-info.is-inverted{background-color:#fff;color:#3298dc}.button.is-info.is-inverted.is-hovered,.button.is-info.is-inverted:hover{background-color:#f2f2f2}.button.is-info.is-inverted[disabled],fieldset[disabled] .button.is-info.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#3298dc}.button.is-info.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-info.is-outlined{background-color:transparent;border-color:#3298dc;color:#3298dc}.button.is-info.is-outlined.is-focused,.button.is-info.is-outlined.is-hovered,.button.is-info.is-outlined:focus,.button.is-info.is-outlined:hover{background-color:#3298dc;border-color:#3298dc;color:#fff}.button.is-info.is-outlined.is-loading::after{border-color:transparent transparent #3298dc #3298dc!important}.button.is-info.is-outlined.is-loading.is-focused::after,.button.is-info.is-outlined.is-loading.is-hovered::after,.button.is-info.is-outlined.is-loading:focus::after,.button.is-info.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-info.is-outlined[disabled],fieldset[disabled] .button.is-info.is-outlined{background-color:transparent;border-color:#3298dc;box-shadow:none;color:#3298dc}.button.is-info.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-info.is-inverted.is-outlined.is-focused,.button.is-info.is-inverted.is-outlined.is-hovered,.button.is-info.is-inverted.is-outlined:focus,.button.is-info.is-inverted.is-outlined:hover{background-color:#fff;color:#3298dc}.button.is-info.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-info.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-info.is-inverted.is-outlined.is-loading:focus::after,.button.is-info.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #3298dc #3298dc!important}.button.is-info.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-info.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-info.is-light{background-color:#eef6fc;color:#1d72aa}.button.is-info.is-light.is-hovered,.button.is-info.is-light:hover{background-color:#e3f1fa;border-color:transparent;color:#1d72aa}.button.is-info.is-light.is-active,.button.is-info.is-light:active{background-color:#d8ebf8;border-color:transparent;color:#1d72aa}.button.is-success{background-color:#48c774;border-color:transparent;color:#fff}.button.is-success.is-hovered,.button.is-success:hover{background-color:#3ec46d;border-color:transparent;color:#fff}.button.is-success.is-focused,.button.is-success:focus{border-color:transparent;color:#fff}.button.is-success.is-focused:not(:active),.button.is-success:focus:not(:active){box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.button.is-success.is-active,.button.is-success:active{background-color:#3abb67;border-color:transparent;color:#fff}.button.is-success[disabled],fieldset[disabled] .button.is-success{background-color:#48c774;border-color:transparent;box-shadow:none}.button.is-success.is-inverted{background-color:#fff;color:#48c774}.button.is-success.is-inverted.is-hovered,.button.is-success.is-inverted:hover{background-color:#f2f2f2}.button.is-success.is-inverted[disabled],fieldset[disabled] .button.is-success.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#48c774}.button.is-success.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-success.is-outlined{background-color:transparent;border-color:#48c774;color:#48c774}.button.is-success.is-outlined.is-focused,.button.is-success.is-outlined.is-hovered,.button.is-success.is-outlined:focus,.button.is-success.is-outlined:hover{background-color:#48c774;border-color:#48c774;color:#fff}.button.is-success.is-outlined.is-loading::after{border-color:transparent transparent #48c774 #48c774!important}.button.is-success.is-outlined.is-loading.is-focused::after,.button.is-success.is-outlined.is-loading.is-hovered::after,.button.is-success.is-outlined.is-loading:focus::after,.button.is-success.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-success.is-outlined[disabled],fieldset[disabled] .button.is-success.is-outlined{background-color:transparent;border-color:#48c774;box-shadow:none;color:#48c774}.button.is-success.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-success.is-inverted.is-outlined.is-focused,.button.is-success.is-inverted.is-outlined.is-hovered,.button.is-success.is-inverted.is-outlined:focus,.button.is-success.is-inverted.is-outlined:hover{background-color:#fff;color:#48c774}.button.is-success.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-success.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-success.is-inverted.is-outlined.is-loading:focus::after,.button.is-success.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #48c774 #48c774!important}.button.is-success.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-success.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-success.is-light{background-color:#effaf3;color:#257942}.button.is-success.is-light.is-hovered,.button.is-success.is-light:hover{background-color:#e6f7ec;border-color:transparent;color:#257942}.button.is-success.is-light.is-active,.button.is-success.is-light:active{background-color:#dcf4e4;border-color:transparent;color:#257942}.button.is-warning{background-color:#ffdd57;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-hovered,.button.is-warning:hover{background-color:#ffdb4a;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-focused,.button.is-warning:focus{border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-focused:not(:active),.button.is-warning:focus:not(:active){box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.button.is-warning.is-active,.button.is-warning:active{background-color:#ffd83d;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning[disabled],fieldset[disabled] .button.is-warning{background-color:#ffdd57;border-color:transparent;box-shadow:none}.button.is-warning.is-inverted{background-color:rgba(0,0,0,.7);color:#ffdd57}.button.is-warning.is-inverted.is-hovered,.button.is-warning.is-inverted:hover{background-color:rgba(0,0,0,.7)}.button.is-warning.is-inverted[disabled],fieldset[disabled] .button.is-warning.is-inverted{background-color:rgba(0,0,0,.7);border-color:transparent;box-shadow:none;color:#ffdd57}.button.is-warning.is-loading::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-warning.is-outlined{background-color:transparent;border-color:#ffdd57;color:#ffdd57}.button.is-warning.is-outlined.is-focused,.button.is-warning.is-outlined.is-hovered,.button.is-warning.is-outlined:focus,.button.is-warning.is-outlined:hover{background-color:#ffdd57;border-color:#ffdd57;color:rgba(0,0,0,.7)}.button.is-warning.is-outlined.is-loading::after{border-color:transparent transparent #ffdd57 #ffdd57!important}.button.is-warning.is-outlined.is-loading.is-focused::after,.button.is-warning.is-outlined.is-loading.is-hovered::after,.button.is-warning.is-outlined.is-loading:focus::after,.button.is-warning.is-outlined.is-loading:hover::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-warning.is-outlined[disabled],fieldset[disabled] .button.is-warning.is-outlined{background-color:transparent;border-color:#ffdd57;box-shadow:none;color:#ffdd57}.button.is-warning.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);color:rgba(0,0,0,.7)}.button.is-warning.is-inverted.is-outlined.is-focused,.button.is-warning.is-inverted.is-outlined.is-hovered,.button.is-warning.is-inverted.is-outlined:focus,.button.is-warning.is-inverted.is-outlined:hover{background-color:rgba(0,0,0,.7);color:#ffdd57}.button.is-warning.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-warning.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-warning.is-inverted.is-outlined.is-loading:focus::after,.button.is-warning.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #ffdd57 #ffdd57!important}.button.is-warning.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-warning.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);box-shadow:none;color:rgba(0,0,0,.7)}.button.is-warning.is-light{background-color:#fffbeb;color:#947600}.button.is-warning.is-light.is-hovered,.button.is-warning.is-light:hover{background-color:#fff8de;border-color:transparent;color:#947600}.button.is-warning.is-light.is-active,.button.is-warning.is-light:active{background-color:#fff6d1;border-color:transparent;color:#947600}.button.is-danger{background-color:#f14668;border-color:transparent;color:#fff}.button.is-danger.is-hovered,.button.is-danger:hover{background-color:#f03a5f;border-color:transparent;color:#fff}.button.is-danger.is-focused,.button.is-danger:focus{border-color:transparent;color:#fff}.button.is-danger.is-focused:not(:active),.button.is-danger:focus:not(:active){box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.button.is-danger.is-active,.button.is-danger:active{background-color:#ef2e55;border-color:transparent;color:#fff}.button.is-danger[disabled],fieldset[disabled] .button.is-danger{background-color:#f14668;border-color:transparent;box-shadow:none}.button.is-danger.is-inverted{background-color:#fff;color:#f14668}.button.is-danger.is-inverted.is-hovered,.button.is-danger.is-inverted:hover{background-color:#f2f2f2}.button.is-danger.is-inverted[disabled],fieldset[disabled] .button.is-danger.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#f14668}.button.is-danger.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-danger.is-outlined{background-color:transparent;border-color:#f14668;color:#f14668}.button.is-danger.is-outlined.is-focused,.button.is-danger.is-outlined.is-hovered,.button.is-danger.is-outlined:focus,.button.is-danger.is-outlined:hover{background-color:#f14668;border-color:#f14668;color:#fff}.button.is-danger.is-outlined.is-loading::after{border-color:transparent transparent #f14668 #f14668!important}.button.is-danger.is-outlined.is-loading.is-focused::after,.button.is-danger.is-outlined.is-loading.is-hovered::after,.button.is-danger.is-outlined.is-loading:focus::after,.button.is-danger.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-danger.is-outlined[disabled],fieldset[disabled] .button.is-danger.is-outlined{background-color:transparent;border-color:#f14668;box-shadow:none;color:#f14668}.button.is-danger.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-danger.is-inverted.is-outlined.is-focused,.button.is-danger.is-inverted.is-outlined.is-hovered,.button.is-danger.is-inverted.is-outlined:focus,.button.is-danger.is-inverted.is-outlined:hover{background-color:#fff;color:#f14668}.button.is-danger.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-danger.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-danger.is-inverted.is-outlined.is-loading:focus::after,.button.is-danger.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #f14668 #f14668!important}.button.is-danger.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-danger.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-danger.is-light{background-color:#feecf0;color:#cc0f35}.button.is-danger.is-light.is-hovered,.button.is-danger.is-light:hover{background-color:#fde0e6;border-color:transparent;color:#cc0f35}.button.is-danger.is-light.is-active,.button.is-danger.is-light:active{background-color:#fcd4dc;border-color:transparent;color:#cc0f35}.button.is-small{border-radius:2px;font-size:.75rem}.button.is-normal{font-size:1rem}.button.is-medium{font-size:1.25rem}.button.is-large{font-size:1.5rem}.button[disabled],fieldset[disabled] .button{background-color:#fff;border-color:#dbdbdb;box-shadow:none;opacity:.5}.button.is-fullwidth{display:flex;width:100%}.button.is-loading{color:transparent!important;pointer-events:none}.button.is-loading::after{position:absolute;left:calc(50% - (1em / 2));top:calc(50% - (1em / 2));position:absolute!important}.button.is-static{background-color:#f5f5f5;border-color:#dbdbdb;color:#7a7a7a;box-shadow:none;pointer-events:none}.button.is-rounded{border-radius:290486px;padding-left:calc(1em + .25em);padding-right:calc(1em + .25em)}.buttons{align-items:center;display:flex;flex-wrap:wrap;justify-content:flex-start}.buttons .button{margin-bottom:.5rem}.buttons .button:not(:last-child):not(.is-fullwidth){margin-right:.5rem}.buttons:last-child{margin-bottom:-.5rem}.buttons:not(:last-child){margin-bottom:1rem}.buttons.are-small .button:not(.is-normal):not(.is-medium):not(.is-large){border-radius:2px;font-size:.75rem}.buttons.are-medium .button:not(.is-small):not(.is-normal):not(.is-large){font-size:1.25rem}.buttons.are-large .button:not(.is-small):not(.is-normal):not(.is-medium){font-size:1.5rem}.buttons.has-addons .button:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.buttons.has-addons .button:not(:last-child){border-bottom-right-radius:0;border-top-right-radius:0;margin-right:-1px}.buttons.has-addons .button:last-child{margin-right:0}.buttons.has-addons .button.is-hovered,.buttons.has-addons .button:hover{z-index:2}.buttons.has-addons .button.is-active,.buttons.has-addons .button.is-focused,.buttons.has-addons .button.is-selected,.buttons.has-addons .button:active,.buttons.has-addons .button:focus{z-index:3}.buttons.has-addons .button.is-active:hover,.buttons.has-addons .button.is-focused:hover,.buttons.has-addons .button.is-selected:hover,.buttons.has-addons .button:active:hover,.buttons.has-addons .button:focus:hover{z-index:4}.buttons.has-addons .button.is-expanded{flex-grow:1;flex-shrink:1}.buttons.is-centered{justify-content:center}.buttons.is-centered:not(.has-addons) .button:not(.is-fullwidth){margin-left:.25rem;margin-right:.25rem}.buttons.is-right{justify-content:flex-end}.buttons.is-right:not(.has-addons) .button:not(.is-fullwidth){margin-left:.25rem;margin-right:.25rem}.container{flex-grow:1;margin:0 auto;position:relative;width:auto}.container.is-fluid{max-width:none;padding-left:32px;padding-right:32px;width:100%}@media screen and (min-width:1024px){.container{max-width:960px}}@media screen and (max-width:1215px){.container.is-widescreen{max-width:1152px}}@media screen and (max-width:1407px){.container.is-fullhd{max-width:1344px}}@media screen and (min-width:1216px){.container{max-width:1152px}}@media screen and (min-width:1408px){.container{max-width:1344px}}.content li+li{margin-top:.25em}.content blockquote:not(:last-child),.content dl:not(:last-child),.content ol:not(:last-child),.content p:not(:last-child),.content pre:not(:last-child),.content table:not(:last-child),.content ul:not(:last-child){margin-bottom:1em}.content h1,.content h2,.content h3,.content h4,.content h5,.content h6{color:#363636;font-weight:600;line-height:1.125}.content h1{font-size:2em;margin-bottom:.5em}.content h1:not(:first-child){margin-top:1em}.content h2{font-size:1.75em;margin-bottom:.5714em}.content h2:not(:first-child){margin-top:1.1428em}.content h3{font-size:1.5em;margin-bottom:.6666em}.content h3:not(:first-child){margin-top:1.3333em}.content h4{font-size:1.25em;margin-bottom:.8em}.content h5{font-size:1.125em;margin-bottom:.8888em}.content h6{font-size:1em;margin-bottom:1em}.content blockquote{background-color:#f5f5f5;border-left:5px solid #dbdbdb;padding:1.25em 1.5em}.content ol{list-style-position:outside;margin-left:2em;margin-top:1em}.content ol:not([type]){list-style-type:decimal}.content ol:not([type]).is-lower-alpha{list-style-type:lower-alpha}.content ol:not([type]).is-lower-roman{list-style-type:lower-roman}.content ol:not([type]).is-upper-alpha{list-style-type:upper-alpha}.content ol:not([type]).is-upper-roman{list-style-type:upper-roman}.content ul{list-style:disc outside;margin-left:2em;margin-top:1em}.content ul ul{list-style-type:circle;margin-top:.5em}.content ul ul ul{list-style-type:square}.content dd{margin-left:2em}.content figure{margin-left:2em;margin-right:2em;text-align:center}.content figure:not(:first-child){margin-top:2em}.content figure:not(:last-child){margin-bottom:2em}.content figure img{display:inline-block}.content figure figcaption{font-style:italic}.content pre{-webkit-overflow-scrolling:touch;overflow-x:auto;padding:1.25em 1.5em;white-space:pre;word-wrap:normal}.content sub,.content sup{font-size:75%}.content table{width:100%}.content table td,.content table th{border:1px solid #dbdbdb;border-width:0 0 1px;padding:.5em .75em;vertical-align:top}.content table th{color:#363636}.content table th:not([align]){text-align:left}.content table thead td,.content table thead th{border-width:0 0 2px;color:#363636}.content table tfoot td,.content table tfoot th{border-width:2px 0 0;color:#363636}.content table tbody tr:last-child td,.content table tbody tr:last-child th{border-bottom-width:0}.content .tabs li+li{margin-top:0}.content.is-small{font-size:.75rem}.content.is-medium{font-size:1.25rem}.content.is-large{font-size:1.5rem}.icon{align-items:center;display:inline-flex;justify-content:center;height:1.5rem;width:1.5rem}.icon.is-small{height:1rem;width:1rem}.icon.is-medium{height:2rem;width:2rem}.icon.is-large{height:3rem;width:3rem}.image{display:block;position:relative}.image img{display:block;height:auto;width:100%}.image img.is-rounded{border-radius:290486px}.image.is-fullwidth{width:100%}.image.is-16by9 .has-ratio,.image.is-16by9 img,.image.is-1by1 .has-ratio,.image.is-1by1 img,.image.is-1by2 .has-ratio,.image.is-1by2 img,.image.is-1by3 .has-ratio,.image.is-1by3 img,.image.is-2by1 .has-ratio,.image.is-2by1 img,.image.is-2by3 .has-ratio,.image.is-2by3 img,.image.is-3by1 .has-ratio,.image.is-3by1 img,.image.is-3by2 .has-ratio,.image.is-3by2 img,.image.is-3by4 .has-ratio,.image.is-3by4 img,.image.is-3by5 .has-ratio,.image.is-3by5 img,.image.is-4by3 .has-ratio,.image.is-4by3 img,.image.is-4by5 .has-ratio,.image.is-4by5 img,.image.is-5by3 .has-ratio,.image.is-5by3 img,.image.is-5by4 .has-ratio,.image.is-5by4 img,.image.is-9by16 .has-ratio,.image.is-9by16 img,.image.is-square .has-ratio,.image.is-square img{height:100%;width:100%}.image.is-1by1,.image.is-square{padding-top:100%}.image.is-5by4{padding-top:80%}.image.is-4by3{padding-top:75%}.image.is-3by2{padding-top:66.6666%}.image.is-5by3{padding-top:60%}.image.is-16by9{padding-top:56.25%}.image.is-2by1{padding-top:50%}.image.is-3by1{padding-top:33.3333%}.image.is-4by5{padding-top:125%}.image.is-3by4{padding-top:133.3333%}.image.is-2by3{padding-top:150%}.image.is-3by5{padding-top:166.6666%}.image.is-9by16{padding-top:177.7777%}.image.is-1by2{padding-top:200%}.image.is-1by3{padding-top:300%}.image.is-16x16{height:16px;width:16px}.image.is-24x24{height:24px;width:24px}.image.is-32x32{height:32px;width:32px}.image.is-48x48{height:48px;width:48px}.image.is-64x64{height:64px;width:64px}.image.is-96x96{height:96px;width:96px}.image.is-128x128{height:128px;width:128px}.notification{background-color:#f5f5f5;border-radius:4px;padding:1.25rem 2.5rem 1.25rem 1.5rem;position:relative}.notification a:not(.button):not(.dropdown-item){color:currentColor;text-decoration:underline}.notification strong{color:currentColor}.notification code,.notification pre{background:#fff}.notification pre code{background:0 0}.notification>.delete{position:absolute;right:.5rem;top:.5rem}.notification .content,.notification .subtitle,.notification .title{color:currentColor}.notification.is-white{background-color:#fff;color:#0a0a0a}.notification.is-black{background-color:#0a0a0a;color:#fff}.notification.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.notification.is-dark{background-color:#363636;color:#fff}.notification.is-primary{background-color:#00d1b2;color:#fff}.notification.is-link{background-color:#3273dc;color:#fff}.notification.is-info{background-color:#3298dc;color:#fff}.notification.is-success{background-color:#48c774;color:#fff}.notification.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.notification.is-danger{background-color:#f14668;color:#fff}.progress{-moz-appearance:none;-webkit-appearance:none;border:none;border-radius:290486px;display:block;height:1rem;overflow:hidden;padding:0;width:100%}.progress::-webkit-progress-bar{background-color:#ededed}.progress::-webkit-progress-value{background-color:#4a4a4a}.progress::-moz-progress-bar{background-color:#4a4a4a}.progress::-ms-fill{background-color:#4a4a4a;border:none}.progress.is-white::-webkit-progress-value{background-color:#fff}.progress.is-white::-moz-progress-bar{background-color:#fff}.progress.is-white::-ms-fill{background-color:#fff}.progress.is-white:indeterminate{background-image:linear-gradient(to right,#fff 30%,#ededed 30%)}.progress.is-black::-webkit-progress-value{background-color:#0a0a0a}.progress.is-black::-moz-progress-bar{background-color:#0a0a0a}.progress.is-black::-ms-fill{background-color:#0a0a0a}.progress.is-black:indeterminate{background-image:linear-gradient(to right,#0a0a0a 30%,#ededed 30%)}.progress.is-light::-webkit-progress-value{background-color:#f5f5f5}.progress.is-light::-moz-progress-bar{background-color:#f5f5f5}.progress.is-light::-ms-fill{background-color:#f5f5f5}.progress.is-light:indeterminate{background-image:linear-gradient(to right,#f5f5f5 30%,#ededed 30%)}.progress.is-dark::-webkit-progress-value{background-color:#363636}.progress.is-dark::-moz-progress-bar{background-color:#363636}.progress.is-dark::-ms-fill{background-color:#363636}.progress.is-dark:indeterminate{background-image:linear-gradient(to right,#363636 30%,#ededed 30%)}.progress.is-primary::-webkit-progress-value{background-color:#00d1b2}.progress.is-primary::-moz-progress-bar{background-color:#00d1b2}.progress.is-primary::-ms-fill{background-color:#00d1b2}.progress.is-primary:indeterminate{background-image:linear-gradient(to right,#00d1b2 30%,#ededed 30%)}.progress.is-link::-webkit-progress-value{background-color:#3273dc}.progress.is-link::-moz-progress-bar{background-color:#3273dc}.progress.is-link::-ms-fill{background-color:#3273dc}.progress.is-link:indeterminate{background-image:linear-gradient(to right,#3273dc 30%,#ededed 30%)}.progress.is-info::-webkit-progress-value{background-color:#3298dc}.progress.is-info::-moz-progress-bar{background-color:#3298dc}.progress.is-info::-ms-fill{background-color:#3298dc}.progress.is-info:indeterminate{background-image:linear-gradient(to right,#3298dc 30%,#ededed 30%)}.progress.is-success::-webkit-progress-value{background-color:#48c774}.progress.is-success::-moz-progress-bar{background-color:#48c774}.progress.is-success::-ms-fill{background-color:#48c774}.progress.is-success:indeterminate{background-image:linear-gradient(to right,#48c774 30%,#ededed 30%)}.progress.is-warning::-webkit-progress-value{background-color:#ffdd57}.progress.is-warning::-moz-progress-bar{background-color:#ffdd57}.progress.is-warning::-ms-fill{background-color:#ffdd57}.progress.is-warning:indeterminate{background-image:linear-gradient(to right,#ffdd57 30%,#ededed 30%)}.progress.is-danger::-webkit-progress-value{background-color:#f14668}.progress.is-danger::-moz-progress-bar{background-color:#f14668}.progress.is-danger::-ms-fill{background-color:#f14668}.progress.is-danger:indeterminate{background-image:linear-gradient(to right,#f14668 30%,#ededed 30%)}.progress:indeterminate{-webkit-animation-duration:1.5s;animation-duration:1.5s;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;-webkit-animation-name:moveIndeterminate;animation-name:moveIndeterminate;-webkit-animation-timing-function:linear;animation-timing-function:linear;background-color:#ededed;background-image:linear-gradient(to right,#4a4a4a 30%,#ededed 30%);background-position:top left;background-repeat:no-repeat;background-size:150% 150%}.progress:indeterminate::-webkit-progress-bar{background-color:transparent}.progress:indeterminate::-moz-progress-bar{background-color:transparent}.progress.is-small{height:.75rem}.progress.is-medium{height:1.25rem}.progress.is-large{height:1.5rem}@-webkit-keyframes moveIndeterminate{from{background-position:200% 0}to{background-position:-200% 0}}@keyframes moveIndeterminate{from{background-position:200% 0}to{background-position:-200% 0}}.table{background-color:#fff;color:#363636}.table td,.table th{border:1px solid #dbdbdb;border-width:0 0 1px;padding:.5em .75em;vertical-align:top}.table td.is-white,.table th.is-white{background-color:#fff;border-color:#fff;color:#0a0a0a}.table td.is-black,.table th.is-black{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.table td.is-light,.table th.is-light{background-color:#f5f5f5;border-color:#f5f5f5;color:rgba(0,0,0,.7)}.table td.is-dark,.table th.is-dark{background-color:#363636;border-color:#363636;color:#fff}.table td.is-primary,.table th.is-primary{background-color:#00d1b2;border-color:#00d1b2;color:#fff}.table td.is-link,.table th.is-link{background-color:#3273dc;border-color:#3273dc;color:#fff}.table td.is-info,.table th.is-info{background-color:#3298dc;border-color:#3298dc;color:#fff}.table td.is-success,.table th.is-success{background-color:#48c774;border-color:#48c774;color:#fff}.table td.is-warning,.table th.is-warning{background-color:#ffdd57;border-color:#ffdd57;color:rgba(0,0,0,.7)}.table td.is-danger,.table th.is-danger{background-color:#f14668;border-color:#f14668;color:#fff}.table td.is-narrow,.table th.is-narrow{white-space:nowrap;width:1%}.table td.is-selected,.table th.is-selected{background-color:#00d1b2;color:#fff}.table td.is-selected a,.table td.is-selected strong,.table th.is-selected a,.table th.is-selected strong{color:currentColor}.table th{color:#363636}.table th:not([align]){text-align:left}.table tr.is-selected{background-color:#00d1b2;color:#fff}.table tr.is-selected a,.table tr.is-selected strong{color:currentColor}.table tr.is-selected td,.table tr.is-selected th{border-color:#fff;color:currentColor}.table thead{background-color:transparent}.table thead td,.table thead th{border-width:0 0 2px;color:#363636}.table tfoot{background-color:transparent}.table tfoot td,.table tfoot th{border-width:2px 0 0;color:#363636}.table tbody{background-color:transparent}.table tbody tr:last-child td,.table tbody tr:last-child th{border-bottom-width:0}.table.is-bordered td,.table.is-bordered th{border-width:1px}.table.is-bordered tr:last-child td,.table.is-bordered tr:last-child th{border-bottom-width:1px}.table.is-fullwidth{width:100%}.table.is-hoverable tbody tr:not(.is-selected):hover{background-color:#fafafa}.table.is-hoverable.is-striped tbody tr:not(.is-selected):hover{background-color:#fafafa}.table.is-hoverable.is-striped tbody tr:not(.is-selected):hover:nth-child(even){background-color:#f5f5f5}.table.is-narrow td,.table.is-narrow th{padding:.25em .5em}.table.is-striped tbody tr:not(.is-selected):nth-child(even){background-color:#fafafa}.table-container{-webkit-overflow-scrolling:touch;overflow:auto;overflow-y:hidden;max-width:100%}.tags{align-items:center;display:flex;flex-wrap:wrap;justify-content:flex-start}.tags .tag{margin-bottom:.5rem}.tags .tag:not(:last-child){margin-right:.5rem}.tags:last-child{margin-bottom:-.5rem}.tags:not(:last-child){margin-bottom:1rem}.tags.are-medium .tag:not(.is-normal):not(.is-large){font-size:1rem}.tags.are-large .tag:not(.is-normal):not(.is-medium){font-size:1.25rem}.tags.is-centered{justify-content:center}.tags.is-centered .tag{margin-right:.25rem;margin-left:.25rem}.tags.is-right{justify-content:flex-end}.tags.is-right .tag:not(:first-child){margin-left:.5rem}.tags.is-right .tag:not(:last-child){margin-right:0}.tags.has-addons .tag{margin-right:0}.tags.has-addons .tag:not(:first-child){margin-left:0;border-bottom-left-radius:0;border-top-left-radius:0}.tags.has-addons .tag:not(:last-child){border-bottom-right-radius:0;border-top-right-radius:0}.tag:not(body){align-items:center;background-color:#f5f5f5;border-radius:4px;color:#4a4a4a;display:inline-flex;font-size:.75rem;height:2em;justify-content:center;line-height:1.5;padding-left:.75em;padding-right:.75em;white-space:nowrap}.tag:not(body) .delete{margin-left:.25rem;margin-right:-.375rem}.tag:not(body).is-white{background-color:#fff;color:#0a0a0a}.tag:not(body).is-black{background-color:#0a0a0a;color:#fff}.tag:not(body).is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.tag:not(body).is-dark{background-color:#363636;color:#fff}.tag:not(body).is-primary{background-color:#00d1b2;color:#fff}.tag:not(body).is-primary.is-light{background-color:#ebfffc;color:#00947e}.tag:not(body).is-link{background-color:#3273dc;color:#fff}.tag:not(body).is-link.is-light{background-color:#eef3fc;color:#2160c4}.tag:not(body).is-info{background-color:#3298dc;color:#fff}.tag:not(body).is-info.is-light{background-color:#eef6fc;color:#1d72aa}.tag:not(body).is-success{background-color:#48c774;color:#fff}.tag:not(body).is-success.is-light{background-color:#effaf3;color:#257942}.tag:not(body).is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.tag:not(body).is-warning.is-light{background-color:#fffbeb;color:#947600}.tag:not(body).is-danger{background-color:#f14668;color:#fff}.tag:not(body).is-danger.is-light{background-color:#feecf0;color:#cc0f35}.tag:not(body).is-normal{font-size:.75rem}.tag:not(body).is-medium{font-size:1rem}.tag:not(body).is-large{font-size:1.25rem}.tag:not(body) .icon:first-child:not(:last-child){margin-left:-.375em;margin-right:.1875em}.tag:not(body) .icon:last-child:not(:first-child){margin-left:.1875em;margin-right:-.375em}.tag:not(body) .icon:first-child:last-child{margin-left:-.375em;margin-right:-.375em}.tag:not(body).is-delete{margin-left:1px;padding:0;position:relative;width:2em}.tag:not(body).is-delete::after,.tag:not(body).is-delete::before{background-color:currentColor;content:"";display:block;left:50%;position:absolute;top:50%;transform:translateX(-50%) translateY(-50%) rotate(45deg);transform-origin:center center}.tag:not(body).is-delete::before{height:1px;width:50%}.tag:not(body).is-delete::after{height:50%;width:1px}.tag:not(body).is-delete:focus,.tag:not(body).is-delete:hover{background-color:#e8e8e8}.tag:not(body).is-delete:active{background-color:#dbdbdb}.tag:not(body).is-rounded{border-radius:290486px}a.tag:hover{text-decoration:underline}.subtitle,.title{word-break:break-word}.subtitle em,.subtitle span,.title em,.title span{font-weight:inherit}.subtitle sub,.title sub{font-size:.75em}.subtitle sup,.title sup{font-size:.75em}.subtitle .tag,.title .tag{vertical-align:middle}.title{color:#363636;font-size:2rem;font-weight:600;line-height:1.125}.title strong{color:inherit;font-weight:inherit}.title+.highlight{margin-top:-.75rem}.title:not(.is-spaced)+.subtitle{margin-top:-1.25rem}.title.is-1{font-size:3rem}.title.is-2{font-size:2.5rem}.title.is-3{font-size:2rem}.title.is-4{font-size:1.5rem}.title.is-5{font-size:1.25rem}.title.is-6{font-size:1rem}.title.is-7{font-size:.75rem}.subtitle{color:#4a4a4a;font-size:1.25rem;font-weight:400;line-height:1.25}.subtitle strong{color:#363636;font-weight:600}.subtitle:not(.is-spaced)+.title{margin-top:-1.25rem}.subtitle.is-1{font-size:3rem}.subtitle.is-2{font-size:2.5rem}.subtitle.is-3{font-size:2rem}.subtitle.is-4{font-size:1.5rem}.subtitle.is-5{font-size:1.25rem}.subtitle.is-6{font-size:1rem}.subtitle.is-7{font-size:.75rem}.heading{display:block;font-size:11px;letter-spacing:1px;margin-bottom:5px;text-transform:uppercase}.highlight{font-weight:400;max-width:100%;overflow:hidden;padding:0}.highlight pre{overflow:auto;max-width:100%}.number{align-items:center;background-color:#f5f5f5;border-radius:290486px;display:inline-flex;font-size:1.25rem;height:2em;justify-content:center;margin-right:1.5rem;min-width:2.5em;padding:.25rem .5rem;text-align:center;vertical-align:top}.input,.select select,.textarea{background-color:#fff;border-color:#dbdbdb;border-radius:4px;color:#363636}.input::-moz-placeholder,.select select::-moz-placeholder,.textarea::-moz-placeholder{color:rgba(54,54,54,.3)}.input::-webkit-input-placeholder,.select select::-webkit-input-placeholder,.textarea::-webkit-input-placeholder{color:rgba(54,54,54,.3)}.input:-moz-placeholder,.select select:-moz-placeholder,.textarea:-moz-placeholder{color:rgba(54,54,54,.3)}.input:-ms-input-placeholder,.select select:-ms-input-placeholder,.textarea:-ms-input-placeholder{color:rgba(54,54,54,.3)}.input:hover,.is-hovered.input,.is-hovered.textarea,.select select.is-hovered,.select select:hover,.textarea:hover{border-color:#b5b5b5}.input:active,.input:focus,.is-active.input,.is-active.textarea,.is-focused.input,.is-focused.textarea,.select select.is-active,.select select.is-focused,.select select:active,.select select:focus,.textarea:active,.textarea:focus{border-color:#3273dc;box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.input[disabled],.select fieldset[disabled] select,.select select[disabled],.textarea[disabled],fieldset[disabled] .input,fieldset[disabled] .select select,fieldset[disabled] .textarea{background-color:#f5f5f5;border-color:#f5f5f5;box-shadow:none;color:#7a7a7a}.input[disabled]::-moz-placeholder,.select fieldset[disabled] select::-moz-placeholder,.select select[disabled]::-moz-placeholder,.textarea[disabled]::-moz-placeholder,fieldset[disabled] .input::-moz-placeholder,fieldset[disabled] .select select::-moz-placeholder,fieldset[disabled] .textarea::-moz-placeholder{color:rgba(122,122,122,.3)}.input[disabled]::-webkit-input-placeholder,.select fieldset[disabled] select::-webkit-input-placeholder,.select select[disabled]::-webkit-input-placeholder,.textarea[disabled]::-webkit-input-placeholder,fieldset[disabled] .input::-webkit-input-placeholder,fieldset[disabled] .select select::-webkit-input-placeholder,fieldset[disabled] .textarea::-webkit-input-placeholder{color:rgba(122,122,122,.3)}.input[disabled]:-moz-placeholder,.select fieldset[disabled] select:-moz-placeholder,.select select[disabled]:-moz-placeholder,.textarea[disabled]:-moz-placeholder,fieldset[disabled] .input:-moz-placeholder,fieldset[disabled] .select select:-moz-placeholder,fieldset[disabled] .textarea:-moz-placeholder{color:rgba(122,122,122,.3)}.input[disabled]:-ms-input-placeholder,.select fieldset[disabled] select:-ms-input-placeholder,.select select[disabled]:-ms-input-placeholder,.textarea[disabled]:-ms-input-placeholder,fieldset[disabled] .input:-ms-input-placeholder,fieldset[disabled] .select select:-ms-input-placeholder,fieldset[disabled] .textarea:-ms-input-placeholder{color:rgba(122,122,122,.3)}.input,.textarea{box-shadow:inset 0 .0625em .125em rgba(10,10,10,.05);max-width:100%;width:100%}.input[readonly],.textarea[readonly]{box-shadow:none}.is-white.input,.is-white.textarea{border-color:#fff}.is-white.input:active,.is-white.input:focus,.is-white.is-active.input,.is-white.is-active.textarea,.is-white.is-focused.input,.is-white.is-focused.textarea,.is-white.textarea:active,.is-white.textarea:focus{box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.is-black.input,.is-black.textarea{border-color:#0a0a0a}.is-black.input:active,.is-black.input:focus,.is-black.is-active.input,.is-black.is-active.textarea,.is-black.is-focused.input,.is-black.is-focused.textarea,.is-black.textarea:active,.is-black.textarea:focus{box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.is-light.input,.is-light.textarea{border-color:#f5f5f5}.is-light.input:active,.is-light.input:focus,.is-light.is-active.input,.is-light.is-active.textarea,.is-light.is-focused.input,.is-light.is-focused.textarea,.is-light.textarea:active,.is-light.textarea:focus{box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.is-dark.input,.is-dark.textarea{border-color:#363636}.is-dark.input:active,.is-dark.input:focus,.is-dark.is-active.input,.is-dark.is-active.textarea,.is-dark.is-focused.input,.is-dark.is-focused.textarea,.is-dark.textarea:active,.is-dark.textarea:focus{box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.is-primary.input,.is-primary.textarea{border-color:#00d1b2}.is-primary.input:active,.is-primary.input:focus,.is-primary.is-active.input,.is-primary.is-active.textarea,.is-primary.is-focused.input,.is-primary.is-focused.textarea,.is-primary.textarea:active,.is-primary.textarea:focus{box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.is-link.input,.is-link.textarea{border-color:#3273dc}.is-link.input:active,.is-link.input:focus,.is-link.is-active.input,.is-link.is-active.textarea,.is-link.is-focused.input,.is-link.is-focused.textarea,.is-link.textarea:active,.is-link.textarea:focus{box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.is-info.input,.is-info.textarea{border-color:#3298dc}.is-info.input:active,.is-info.input:focus,.is-info.is-active.input,.is-info.is-active.textarea,.is-info.is-focused.input,.is-info.is-focused.textarea,.is-info.textarea:active,.is-info.textarea:focus{box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.is-success.input,.is-success.textarea{border-color:#48c774}.is-success.input:active,.is-success.input:focus,.is-success.is-active.input,.is-success.is-active.textarea,.is-success.is-focused.input,.is-success.is-focused.textarea,.is-success.textarea:active,.is-success.textarea:focus{box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.is-warning.input,.is-warning.textarea{border-color:#ffdd57}.is-warning.input:active,.is-warning.input:focus,.is-warning.is-active.input,.is-warning.is-active.textarea,.is-warning.is-focused.input,.is-warning.is-focused.textarea,.is-warning.textarea:active,.is-warning.textarea:focus{box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.is-danger.input,.is-danger.textarea{border-color:#f14668}.is-danger.input:active,.is-danger.input:focus,.is-danger.is-active.input,.is-danger.is-active.textarea,.is-danger.is-focused.input,.is-danger.is-focused.textarea,.is-danger.textarea:active,.is-danger.textarea:focus{box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.is-small.input,.is-small.textarea{border-radius:2px;font-size:.75rem}.is-medium.input,.is-medium.textarea{font-size:1.25rem}.is-large.input,.is-large.textarea{font-size:1.5rem}.is-fullwidth.input,.is-fullwidth.textarea{display:block;width:100%}.is-inline.input,.is-inline.textarea{display:inline;width:auto}.input.is-rounded{border-radius:290486px;padding-left:calc(calc(.75em - 1px) + .375em);padding-right:calc(calc(.75em - 1px) + .375em)}.input.is-static{background-color:transparent;border-color:transparent;box-shadow:none;padding-left:0;padding-right:0}.textarea{display:block;max-width:100%;min-width:100%;padding:calc(.75em - 1px);resize:vertical}.textarea:not([rows]){max-height:40em;min-height:8em}.textarea[rows]{height:initial}.textarea.has-fixed-size{resize:none}.checkbox,.radio{cursor:pointer;display:inline-block;line-height:1.25;position:relative}.checkbox input,.radio input{cursor:pointer}.checkbox:hover,.radio:hover{color:#363636}.checkbox[disabled],.radio[disabled],fieldset[disabled] .checkbox,fieldset[disabled] .radio{color:#7a7a7a;cursor:not-allowed}.radio+.radio{margin-left:.5em}.select{display:inline-block;max-width:100%;position:relative;vertical-align:top}.select:not(.is-multiple){height:2.5em}.select:not(.is-multiple):not(.is-loading)::after{border-color:#3273dc;right:1.125em;z-index:4}.select.is-rounded select{border-radius:290486px;padding-left:1em}.select select{cursor:pointer;display:block;font-size:1em;max-width:100%;outline:0}.select select::-ms-expand{display:none}.select select[disabled]:hover,fieldset[disabled] .select select:hover{border-color:#f5f5f5}.select select:not([multiple]){padding-right:2.5em}.select select[multiple]{height:auto;padding:0}.select select[multiple] option{padding:.5em 1em}.select:not(.is-multiple):not(.is-loading):hover::after{border-color:#363636}.select.is-white:not(:hover)::after{border-color:#fff}.select.is-white select{border-color:#fff}.select.is-white select.is-hovered,.select.is-white select:hover{border-color:#f2f2f2}.select.is-white select.is-active,.select.is-white select.is-focused,.select.is-white select:active,.select.is-white select:focus{box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.select.is-black:not(:hover)::after{border-color:#0a0a0a}.select.is-black select{border-color:#0a0a0a}.select.is-black select.is-hovered,.select.is-black select:hover{border-color:#000}.select.is-black select.is-active,.select.is-black select.is-focused,.select.is-black select:active,.select.is-black select:focus{box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.select.is-light:not(:hover)::after{border-color:#f5f5f5}.select.is-light select{border-color:#f5f5f5}.select.is-light select.is-hovered,.select.is-light select:hover{border-color:#e8e8e8}.select.is-light select.is-active,.select.is-light select.is-focused,.select.is-light select:active,.select.is-light select:focus{box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.select.is-dark:not(:hover)::after{border-color:#363636}.select.is-dark select{border-color:#363636}.select.is-dark select.is-hovered,.select.is-dark select:hover{border-color:#292929}.select.is-dark select.is-active,.select.is-dark select.is-focused,.select.is-dark select:active,.select.is-dark select:focus{box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.select.is-primary:not(:hover)::after{border-color:#00d1b2}.select.is-primary select{border-color:#00d1b2}.select.is-primary select.is-hovered,.select.is-primary select:hover{border-color:#00b89c}.select.is-primary select.is-active,.select.is-primary select.is-focused,.select.is-primary select:active,.select.is-primary select:focus{box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.select.is-link:not(:hover)::after{border-color:#3273dc}.select.is-link select{border-color:#3273dc}.select.is-link select.is-hovered,.select.is-link select:hover{border-color:#2366d1}.select.is-link select.is-active,.select.is-link select.is-focused,.select.is-link select:active,.select.is-link select:focus{box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.select.is-info:not(:hover)::after{border-color:#3298dc}.select.is-info select{border-color:#3298dc}.select.is-info select.is-hovered,.select.is-info select:hover{border-color:#238cd1}.select.is-info select.is-active,.select.is-info select.is-focused,.select.is-info select:active,.select.is-info select:focus{box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.select.is-success:not(:hover)::after{border-color:#48c774}.select.is-success select{border-color:#48c774}.select.is-success select.is-hovered,.select.is-success select:hover{border-color:#3abb67}.select.is-success select.is-active,.select.is-success select.is-focused,.select.is-success select:active,.select.is-success select:focus{box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.select.is-warning:not(:hover)::after{border-color:#ffdd57}.select.is-warning select{border-color:#ffdd57}.select.is-warning select.is-hovered,.select.is-warning select:hover{border-color:#ffd83d}.select.is-warning select.is-active,.select.is-warning select.is-focused,.select.is-warning select:active,.select.is-warning select:focus{box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.select.is-danger:not(:hover)::after{border-color:#f14668}.select.is-danger select{border-color:#f14668}.select.is-danger select.is-hovered,.select.is-danger select:hover{border-color:#ef2e55}.select.is-danger select.is-active,.select.is-danger select.is-focused,.select.is-danger select:active,.select.is-danger select:focus{box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.select.is-small{border-radius:2px;font-size:.75rem}.select.is-medium{font-size:1.25rem}.select.is-large{font-size:1.5rem}.select.is-disabled::after{border-color:#7a7a7a}.select.is-fullwidth{width:100%}.select.is-fullwidth select{width:100%}.select.is-loading::after{margin-top:0;position:absolute;right:.625em;top:.625em;transform:none}.select.is-loading.is-small:after{font-size:.75rem}.select.is-loading.is-medium:after{font-size:1.25rem}.select.is-loading.is-large:after{font-size:1.5rem}.file{align-items:stretch;display:flex;justify-content:flex-start;position:relative}.file.is-white .file-cta{background-color:#fff;border-color:transparent;color:#0a0a0a}.file.is-white.is-hovered .file-cta,.file.is-white:hover .file-cta{background-color:#f9f9f9;border-color:transparent;color:#0a0a0a}.file.is-white.is-focused .file-cta,.file.is-white:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(255,255,255,.25);color:#0a0a0a}.file.is-white.is-active .file-cta,.file.is-white:active .file-cta{background-color:#f2f2f2;border-color:transparent;color:#0a0a0a}.file.is-black .file-cta{background-color:#0a0a0a;border-color:transparent;color:#fff}.file.is-black.is-hovered .file-cta,.file.is-black:hover .file-cta{background-color:#040404;border-color:transparent;color:#fff}.file.is-black.is-focused .file-cta,.file.is-black:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(10,10,10,.25);color:#fff}.file.is-black.is-active .file-cta,.file.is-black:active .file-cta{background-color:#000;border-color:transparent;color:#fff}.file.is-light .file-cta{background-color:#f5f5f5;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-light.is-hovered .file-cta,.file.is-light:hover .file-cta{background-color:#eee;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-light.is-focused .file-cta,.file.is-light:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(245,245,245,.25);color:rgba(0,0,0,.7)}.file.is-light.is-active .file-cta,.file.is-light:active .file-cta{background-color:#e8e8e8;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-dark .file-cta{background-color:#363636;border-color:transparent;color:#fff}.file.is-dark.is-hovered .file-cta,.file.is-dark:hover .file-cta{background-color:#2f2f2f;border-color:transparent;color:#fff}.file.is-dark.is-focused .file-cta,.file.is-dark:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(54,54,54,.25);color:#fff}.file.is-dark.is-active .file-cta,.file.is-dark:active .file-cta{background-color:#292929;border-color:transparent;color:#fff}.file.is-primary .file-cta{background-color:#00d1b2;border-color:transparent;color:#fff}.file.is-primary.is-hovered .file-cta,.file.is-primary:hover .file-cta{background-color:#00c4a7;border-color:transparent;color:#fff}.file.is-primary.is-focused .file-cta,.file.is-primary:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(0,209,178,.25);color:#fff}.file.is-primary.is-active .file-cta,.file.is-primary:active .file-cta{background-color:#00b89c;border-color:transparent;color:#fff}.file.is-link .file-cta{background-color:#3273dc;border-color:transparent;color:#fff}.file.is-link.is-hovered .file-cta,.file.is-link:hover .file-cta{background-color:#276cda;border-color:transparent;color:#fff}.file.is-link.is-focused .file-cta,.file.is-link:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(50,115,220,.25);color:#fff}.file.is-link.is-active .file-cta,.file.is-link:active .file-cta{background-color:#2366d1;border-color:transparent;color:#fff}.file.is-info .file-cta{background-color:#3298dc;border-color:transparent;color:#fff}.file.is-info.is-hovered .file-cta,.file.is-info:hover .file-cta{background-color:#2793da;border-color:transparent;color:#fff}.file.is-info.is-focused .file-cta,.file.is-info:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(50,152,220,.25);color:#fff}.file.is-info.is-active .file-cta,.file.is-info:active .file-cta{background-color:#238cd1;border-color:transparent;color:#fff}.file.is-success .file-cta{background-color:#48c774;border-color:transparent;color:#fff}.file.is-success.is-hovered .file-cta,.file.is-success:hover .file-cta{background-color:#3ec46d;border-color:transparent;color:#fff}.file.is-success.is-focused .file-cta,.file.is-success:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(72,199,116,.25);color:#fff}.file.is-success.is-active .file-cta,.file.is-success:active .file-cta{background-color:#3abb67;border-color:transparent;color:#fff}.file.is-warning .file-cta{background-color:#ffdd57;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-warning.is-hovered .file-cta,.file.is-warning:hover .file-cta{background-color:#ffdb4a;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-warning.is-focused .file-cta,.file.is-warning:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(255,221,87,.25);color:rgba(0,0,0,.7)}.file.is-warning.is-active .file-cta,.file.is-warning:active .file-cta{background-color:#ffd83d;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-danger .file-cta{background-color:#f14668;border-color:transparent;color:#fff}.file.is-danger.is-hovered .file-cta,.file.is-danger:hover .file-cta{background-color:#f03a5f;border-color:transparent;color:#fff}.file.is-danger.is-focused .file-cta,.file.is-danger:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(241,70,104,.25);color:#fff}.file.is-danger.is-active .file-cta,.file.is-danger:active .file-cta{background-color:#ef2e55;border-color:transparent;color:#fff}.file.is-small{font-size:.75rem}.file.is-medium{font-size:1.25rem}.file.is-medium .file-icon .fa{font-size:21px}.file.is-large{font-size:1.5rem}.file.is-large .file-icon .fa{font-size:28px}.file.has-name .file-cta{border-bottom-right-radius:0;border-top-right-radius:0}.file.has-name .file-name{border-bottom-left-radius:0;border-top-left-radius:0}.file.has-name.is-empty .file-cta{border-radius:4px}.file.has-name.is-empty .file-name{display:none}.file.is-boxed .file-label{flex-direction:column}.file.is-boxed .file-cta{flex-direction:column;height:auto;padding:1em 3em}.file.is-boxed .file-name{border-width:0 1px 1px}.file.is-boxed .file-icon{height:1.5em;width:1.5em}.file.is-boxed .file-icon .fa{font-size:21px}.file.is-boxed.is-small .file-icon .fa{font-size:14px}.file.is-boxed.is-medium .file-icon .fa{font-size:28px}.file.is-boxed.is-large .file-icon .fa{font-size:35px}.file.is-boxed.has-name .file-cta{border-radius:4px 4px 0 0}.file.is-boxed.has-name .file-name{border-radius:0 0 4px 4px;border-width:0 1px 1px}.file.is-centered{justify-content:center}.file.is-fullwidth .file-label{width:100%}.file.is-fullwidth .file-name{flex-grow:1;max-width:none}.file.is-right{justify-content:flex-end}.file.is-right .file-cta{border-radius:0 4px 4px 0}.file.is-right .file-name{border-radius:4px 0 0 4px;border-width:1px 0 1px 1px;order:-1}.file-label{align-items:stretch;display:flex;cursor:pointer;justify-content:flex-start;overflow:hidden;position:relative}.file-label:hover .file-cta{background-color:#eee;color:#363636}.file-label:hover .file-name{border-color:#d5d5d5}.file-label:active .file-cta{background-color:#e8e8e8;color:#363636}.file-label:active .file-name{border-color:#cfcfcf}.file-input{height:100%;left:0;opacity:0;outline:0;position:absolute;top:0;width:100%}.file-cta,.file-name{border-color:#dbdbdb;border-radius:4px;font-size:1em;padding-left:1em;padding-right:1em;white-space:nowrap}.file-cta{background-color:#f5f5f5;color:#4a4a4a}.file-name{border-color:#dbdbdb;border-style:solid;border-width:1px 1px 1px 0;display:block;max-width:16em;overflow:hidden;text-align:left;text-overflow:ellipsis}.file-icon{align-items:center;display:flex;height:1em;justify-content:center;margin-right:.5em;width:1em}.file-icon .fa{font-size:14px}.label{color:#363636;display:block;font-size:1rem;font-weight:700}.label:not(:last-child){margin-bottom:.5em}.label.is-small{font-size:.75rem}.label.is-medium{font-size:1.25rem}.label.is-large{font-size:1.5rem}.help{display:block;font-size:.75rem;margin-top:.25rem}.help.is-white{color:#fff}.help.is-black{color:#0a0a0a}.help.is-light{color:#f5f5f5}.help.is-dark{color:#363636}.help.is-primary{color:#00d1b2}.help.is-link{color:#3273dc}.help.is-info{color:#3298dc}.help.is-success{color:#48c774}.help.is-warning{color:#ffdd57}.help.is-danger{color:#f14668}.field:not(:last-child){margin-bottom:.75rem}.field.has-addons{display:flex;justify-content:flex-start}.field.has-addons .control:not(:last-child){margin-right:-1px}.field.has-addons .control:not(:first-child):not(:last-child) .button,.field.has-addons .control:not(:first-child):not(:last-child) .input,.field.has-addons .control:not(:first-child):not(:last-child) .select select{border-radius:0}.field.has-addons .control:first-child:not(:only-child) .button,.field.has-addons .control:first-child:not(:only-child) .input,.field.has-addons .control:first-child:not(:only-child) .select select{border-bottom-right-radius:0;border-top-right-radius:0}.field.has-addons .control:last-child:not(:only-child) .button,.field.has-addons .control:last-child:not(:only-child) .input,.field.has-addons .control:last-child:not(:only-child) .select select{border-bottom-left-radius:0;border-top-left-radius:0}.field.has-addons .control .button:not([disabled]).is-hovered,.field.has-addons .control .button:not([disabled]):hover,.field.has-addons .control .input:not([disabled]).is-hovered,.field.has-addons .control .input:not([disabled]):hover,.field.has-addons .control .select select:not([disabled]).is-hovered,.field.has-addons .control .select select:not([disabled]):hover{z-index:2}.field.has-addons .control .button:not([disabled]).is-active,.field.has-addons .control .button:not([disabled]).is-focused,.field.has-addons .control .button:not([disabled]):active,.field.has-addons .control .button:not([disabled]):focus,.field.has-addons .control .input:not([disabled]).is-active,.field.has-addons .control .input:not([disabled]).is-focused,.field.has-addons .control .input:not([disabled]):active,.field.has-addons .control .input:not([disabled]):focus,.field.has-addons .control .select select:not([disabled]).is-active,.field.has-addons .control .select select:not([disabled]).is-focused,.field.has-addons .control .select select:not([disabled]):active,.field.has-addons .control .select select:not([disabled]):focus{z-index:3}.field.has-addons .control .button:not([disabled]).is-active:hover,.field.has-addons .control .button:not([disabled]).is-focused:hover,.field.has-addons .control .button:not([disabled]):active:hover,.field.has-addons .control .button:not([disabled]):focus:hover,.field.has-addons .control .input:not([disabled]).is-active:hover,.field.has-addons .control .input:not([disabled]).is-focused:hover,.field.has-addons .control .input:not([disabled]):active:hover,.field.has-addons .control .input:not([disabled]):focus:hover,.field.has-addons .control .select select:not([disabled]).is-active:hover,.field.has-addons .control .select select:not([disabled]).is-focused:hover,.field.has-addons .control .select select:not([disabled]):active:hover,.field.has-addons .control .select select:not([disabled]):focus:hover{z-index:4}.field.has-addons .control.is-expanded{flex-grow:1;flex-shrink:1}.field.has-addons.has-addons-centered{justify-content:center}.field.has-addons.has-addons-right{justify-content:flex-end}.field.has-addons.has-addons-fullwidth .control{flex-grow:1;flex-shrink:0}.field.is-grouped{display:flex;justify-content:flex-start}.field.is-grouped>.control{flex-shrink:0}.field.is-grouped>.control:not(:last-child){margin-bottom:0;margin-right:.75rem}.field.is-grouped>.control.is-expanded{flex-grow:1;flex-shrink:1}.field.is-grouped.is-grouped-centered{justify-content:center}.field.is-grouped.is-grouped-right{justify-content:flex-end}.field.is-grouped.is-grouped-multiline{flex-wrap:wrap}.field.is-grouped.is-grouped-multiline>.control:last-child,.field.is-grouped.is-grouped-multiline>.control:not(:last-child){margin-bottom:.75rem}.field.is-grouped.is-grouped-multiline:last-child{margin-bottom:-.75rem}.field.is-grouped.is-grouped-multiline:not(:last-child){margin-bottom:0}@media screen and (min-width:769px),print{.field.is-horizontal{display:flex}}.field-label .label{font-size:inherit}@media screen and (max-width:768px){.field-label{margin-bottom:.5rem}}@media screen and (min-width:769px),print{.field-label{flex-basis:0;flex-grow:1;flex-shrink:0;margin-right:1.5rem;text-align:right}.field-label.is-small{font-size:.75rem;padding-top:.375em}.field-label.is-normal{padding-top:.375em}.field-label.is-medium{font-size:1.25rem;padding-top:.375em}.field-label.is-large{font-size:1.5rem;padding-top:.375em}}.field-body .field .field{margin-bottom:0}@media screen and (min-width:769px),print{.field-body{display:flex;flex-basis:0;flex-grow:5;flex-shrink:1}.field-body .field{margin-bottom:0}.field-body>.field{flex-shrink:1}.field-body>.field:not(.is-narrow){flex-grow:1}.field-body>.field:not(:last-child){margin-right:.75rem}}.control{box-sizing:border-box;clear:both;font-size:1rem;position:relative;text-align:left}.control.has-icons-left .input:focus~.icon,.control.has-icons-left .select:focus~.icon,.control.has-icons-right .input:focus~.icon,.control.has-icons-right .select:focus~.icon{color:#4a4a4a}.control.has-icons-left .input.is-small~.icon,.control.has-icons-left .select.is-small~.icon,.control.has-icons-right .input.is-small~.icon,.control.has-icons-right .select.is-small~.icon{font-size:.75rem}.control.has-icons-left .input.is-medium~.icon,.control.has-icons-left .select.is-medium~.icon,.control.has-icons-right .input.is-medium~.icon,.control.has-icons-right .select.is-medium~.icon{font-size:1.25rem}.control.has-icons-left .input.is-large~.icon,.control.has-icons-left .select.is-large~.icon,.control.has-icons-right .input.is-large~.icon,.control.has-icons-right .select.is-large~.icon{font-size:1.5rem}.control.has-icons-left .icon,.control.has-icons-right .icon{color:#dbdbdb;height:2.5em;pointer-events:none;position:absolute;top:0;width:2.5em;z-index:4}.control.has-icons-left .input,.control.has-icons-left .select select{padding-left:2.5em}.control.has-icons-left .icon.is-left{left:0}.control.has-icons-right .input,.control.has-icons-right .select select{padding-right:2.5em}.control.has-icons-right .icon.is-right{right:0}.control.is-loading::after{position:absolute!important;right:.625em;top:.625em;z-index:4}.control.is-loading.is-small:after{font-size:.75rem}.control.is-loading.is-medium:after{font-size:1.25rem}.control.is-loading.is-large:after{font-size:1.5rem}.breadcrumb{font-size:1rem;white-space:nowrap}.breadcrumb a{align-items:center;color:#3273dc;display:flex;justify-content:center;padding:0 .75em}.breadcrumb a:hover{color:#363636}.breadcrumb li{align-items:center;display:flex}.breadcrumb li:first-child a{padding-left:0}.breadcrumb li.is-active a{color:#363636;cursor:default;pointer-events:none}.breadcrumb li+li::before{color:#b5b5b5;content:"\0002f"}.breadcrumb ol,.breadcrumb ul{align-items:flex-start;display:flex;flex-wrap:wrap;justify-content:flex-start}.breadcrumb .icon:first-child{margin-right:.5em}.breadcrumb .icon:last-child{margin-left:.5em}.breadcrumb.is-centered ol,.breadcrumb.is-centered ul{justify-content:center}.breadcrumb.is-right ol,.breadcrumb.is-right ul{justify-content:flex-end}.breadcrumb.is-small{font-size:.75rem}.breadcrumb.is-medium{font-size:1.25rem}.breadcrumb.is-large{font-size:1.5rem}.breadcrumb.has-arrow-separator li+li::before{content:"\02192"}.breadcrumb.has-bullet-separator li+li::before{content:"\02022"}.breadcrumb.has-dot-separator li+li::before{content:"\000b7"}.breadcrumb.has-succeeds-separator li+li::before{content:"\0227B"}.card{background-color:#fff;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);color:#4a4a4a;max-width:100%;position:relative}.card-header{background-color:transparent;align-items:stretch;box-shadow:0 .125em .25em rgba(10,10,10,.1);display:flex}.card-header-title{align-items:center;color:#363636;display:flex;flex-grow:1;font-weight:700;padding:.75rem 1rem}.card-header-title.is-centered{justify-content:center}.card-header-icon{align-items:center;cursor:pointer;display:flex;justify-content:center;padding:.75rem 1rem}.card-image{display:block;position:relative}.card-content{background-color:transparent;padding:1.5rem}.card-footer{background-color:transparent;border-top:1px solid #ededed;align-items:stretch;display:flex}.card-footer-item{align-items:center;display:flex;flex-basis:0;flex-grow:1;flex-shrink:0;justify-content:center;padding:.75rem}.card-footer-item:not(:last-child){border-right:1px solid #ededed}.card .media:not(:last-child){margin-bottom:1.5rem}.dropdown{display:inline-flex;position:relative;vertical-align:top}.dropdown.is-active .dropdown-menu,.dropdown.is-hoverable:hover .dropdown-menu{display:block}.dropdown.is-right .dropdown-menu{left:auto;right:0}.dropdown.is-up .dropdown-menu{bottom:100%;padding-bottom:4px;padding-top:initial;top:auto}.dropdown-menu{display:none;left:0;min-width:12rem;padding-top:4px;position:absolute;top:100%;z-index:20}.dropdown-content{background-color:#fff;border-radius:4px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);padding-bottom:.5rem;padding-top:.5rem}.dropdown-item{color:#4a4a4a;display:block;font-size:.875rem;line-height:1.5;padding:.375rem 1rem;position:relative}a.dropdown-item,button.dropdown-item{padding-right:3rem;text-align:left;white-space:nowrap;width:100%}a.dropdown-item:hover,button.dropdown-item:hover{background-color:#f5f5f5;color:#0a0a0a}a.dropdown-item.is-active,button.dropdown-item.is-active{background-color:#3273dc;color:#fff}.dropdown-divider{background-color:#ededed;border:none;display:block;height:1px;margin:.5rem 0}.level{align-items:center;justify-content:space-between}.level code{border-radius:4px}.level img{display:inline-block;vertical-align:top}.level.is-mobile{display:flex}.level.is-mobile .level-left,.level.is-mobile .level-right{display:flex}.level.is-mobile .level-left+.level-right{margin-top:0}.level.is-mobile .level-item:not(:last-child){margin-bottom:0;margin-right:.75rem}.level.is-mobile .level-item:not(.is-narrow){flex-grow:1}@media screen and (min-width:769px),print{.level{display:flex}.level>.level-item:not(.is-narrow){flex-grow:1}}.level-item{align-items:center;display:flex;flex-basis:auto;flex-grow:0;flex-shrink:0;justify-content:center}.level-item .subtitle,.level-item .title{margin-bottom:0}@media screen and (max-width:768px){.level-item:not(:last-child){margin-bottom:.75rem}}.level-left,.level-right{flex-basis:auto;flex-grow:0;flex-shrink:0}.level-left .level-item.is-flexible,.level-right .level-item.is-flexible{flex-grow:1}@media screen and (min-width:769px),print{.level-left .level-item:not(:last-child),.level-right .level-item:not(:last-child){margin-right:.75rem}}.level-left{align-items:center;justify-content:flex-start}@media screen and (max-width:768px){.level-left+.level-right{margin-top:1.5rem}}@media screen and (min-width:769px),print{.level-left{display:flex}}.level-right{align-items:center;justify-content:flex-end}@media screen and (min-width:769px),print{.level-right{display:flex}}.list{background-color:#fff;border-radius:4px;box-shadow:0 2px 3px rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.1)}.list-item{display:block;padding:.5em 1em}.list-item:not(a){color:#4a4a4a}.list-item:first-child{border-top-left-radius:4px;border-top-right-radius:4px}.list-item:last-child{border-bottom-left-radius:4px;border-bottom-right-radius:4px}.list-item:not(:last-child){border-bottom:1px solid #dbdbdb}.list-item.is-active{background-color:#3273dc;color:#fff}a.list-item{background-color:#f5f5f5;cursor:pointer}.media{align-items:flex-start;display:flex;text-align:left}.media .content:not(:last-child){margin-bottom:.75rem}.media .media{border-top:1px solid rgba(219,219,219,.5);display:flex;padding-top:.75rem}.media .media .content:not(:last-child),.media .media .control:not(:last-child){margin-bottom:.5rem}.media .media .media{padding-top:.5rem}.media .media .media+.media{margin-top:.5rem}.media+.media{border-top:1px solid rgba(219,219,219,.5);margin-top:1rem;padding-top:1rem}.media.is-large+.media{margin-top:1.5rem;padding-top:1.5rem}.media-left,.media-right{flex-basis:auto;flex-grow:0;flex-shrink:0}.media-left{margin-right:1rem}.media-right{margin-left:1rem}.media-content{flex-basis:auto;flex-grow:1;flex-shrink:1;text-align:left}@media screen and (max-width:768px){.media-content{overflow-x:auto}}.menu{font-size:1rem}.menu.is-small{font-size:.75rem}.menu.is-medium{font-size:1.25rem}.menu.is-large{font-size:1.5rem}.menu-list{line-height:1.25}.menu-list a{border-radius:2px;color:#4a4a4a;display:block;padding:.5em .75em}.menu-list a:hover{background-color:#f5f5f5;color:#363636}.menu-list a.is-active{background-color:#3273dc;color:#fff}.menu-list li ul{border-left:1px solid #dbdbdb;margin:.75em;padding-left:.75em}.menu-label{color:#7a7a7a;font-size:.75em;letter-spacing:.1em;text-transform:uppercase}.menu-label:not(:first-child){margin-top:1em}.menu-label:not(:last-child){margin-bottom:1em}.message{background-color:#f5f5f5;border-radius:4px;font-size:1rem}.message strong{color:currentColor}.message a:not(.button):not(.tag):not(.dropdown-item){color:currentColor;text-decoration:underline}.message.is-small{font-size:.75rem}.message.is-medium{font-size:1.25rem}.message.is-large{font-size:1.5rem}.message.is-white{background-color:#fff}.message.is-white .message-header{background-color:#fff;color:#0a0a0a}.message.is-white .message-body{border-color:#fff}.message.is-black{background-color:#fafafa}.message.is-black .message-header{background-color:#0a0a0a;color:#fff}.message.is-black .message-body{border-color:#0a0a0a}.message.is-light{background-color:#fafafa}.message.is-light .message-header{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.message.is-light .message-body{border-color:#f5f5f5}.message.is-dark{background-color:#fafafa}.message.is-dark .message-header{background-color:#363636;color:#fff}.message.is-dark .message-body{border-color:#363636}.message.is-primary{background-color:#ebfffc}.message.is-primary .message-header{background-color:#00d1b2;color:#fff}.message.is-primary .message-body{border-color:#00d1b2;color:#00947e}.message.is-link{background-color:#eef3fc}.message.is-link .message-header{background-color:#3273dc;color:#fff}.message.is-link .message-body{border-color:#3273dc;color:#2160c4}.message.is-info{background-color:#eef6fc}.message.is-info .message-header{background-color:#3298dc;color:#fff}.message.is-info .message-body{border-color:#3298dc;color:#1d72aa}.message.is-success{background-color:#effaf3}.message.is-success .message-header{background-color:#48c774;color:#fff}.message.is-success .message-body{border-color:#48c774;color:#257942}.message.is-warning{background-color:#fffbeb}.message.is-warning .message-header{background-color:#ffdd57;color:rgba(0,0,0,.7)}.message.is-warning .message-body{border-color:#ffdd57;color:#947600}.message.is-danger{background-color:#feecf0}.message.is-danger .message-header{background-color:#f14668;color:#fff}.message.is-danger .message-body{border-color:#f14668;color:#cc0f35}.message-header{align-items:center;background-color:#4a4a4a;border-radius:4px 4px 0 0;color:#fff;display:flex;font-weight:700;justify-content:space-between;line-height:1.25;padding:.75em 1em;position:relative}.message-header .delete{flex-grow:0;flex-shrink:0;margin-left:.75em}.message-header+.message-body{border-width:0;border-top-left-radius:0;border-top-right-radius:0}.message-body{border-color:#dbdbdb;border-radius:4px;border-style:solid;border-width:0 0 0 4px;color:#4a4a4a;padding:1.25em 1.5em}.message-body code,.message-body pre{background-color:#fff}.message-body pre code{background-color:transparent}.modal{align-items:center;display:none;flex-direction:column;justify-content:center;overflow:hidden;position:fixed;z-index:40}.modal.is-active{display:flex}.modal-background{background-color:rgba(10,10,10,.86)}.modal-card,.modal-content{margin:0 20px;max-height:calc(100vh - 160px);overflow:auto;position:relative;width:100%}@media screen and (min-width:769px),print{.modal-card,.modal-content{margin:0 auto;max-height:calc(100vh - 40px);width:640px}}.modal-close{background:0 0;height:40px;position:fixed;right:20px;top:20px;width:40px}.modal-card{display:flex;flex-direction:column;max-height:calc(100vh - 40px);overflow:hidden;-ms-overflow-y:visible}.modal-card-foot,.modal-card-head{align-items:center;background-color:#f5f5f5;display:flex;flex-shrink:0;justify-content:flex-start;padding:20px;position:relative}.modal-card-head{border-bottom:1px solid #dbdbdb;border-top-left-radius:6px;border-top-right-radius:6px}.modal-card-title{color:#363636;flex-grow:1;flex-shrink:0;font-size:1.5rem;line-height:1}.modal-card-foot{border-bottom-left-radius:6px;border-bottom-right-radius:6px;border-top:1px solid #dbdbdb}.modal-card-foot .button:not(:last-child){margin-right:.5em}.modal-card-body{-webkit-overflow-scrolling:touch;background-color:#fff;flex-grow:1;flex-shrink:1;overflow:auto;padding:20px}.navbar{background-color:#fff;min-height:3.25rem;position:relative;z-index:30}.navbar.is-white{background-color:#fff;color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link,.navbar.is-white .navbar-brand>.navbar-item{color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link.is-active,.navbar.is-white .navbar-brand .navbar-link:focus,.navbar.is-white .navbar-brand .navbar-link:hover,.navbar.is-white .navbar-brand>a.navbar-item.is-active,.navbar.is-white .navbar-brand>a.navbar-item:focus,.navbar.is-white .navbar-brand>a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link::after{border-color:#0a0a0a}.navbar.is-white .navbar-burger{color:#0a0a0a}@media screen and (min-width:1024px){.navbar.is-white .navbar-end .navbar-link,.navbar.is-white .navbar-end>.navbar-item,.navbar.is-white .navbar-start .navbar-link,.navbar.is-white .navbar-start>.navbar-item{color:#0a0a0a}.navbar.is-white .navbar-end .navbar-link.is-active,.navbar.is-white .navbar-end .navbar-link:focus,.navbar.is-white .navbar-end .navbar-link:hover,.navbar.is-white .navbar-end>a.navbar-item.is-active,.navbar.is-white .navbar-end>a.navbar-item:focus,.navbar.is-white .navbar-end>a.navbar-item:hover,.navbar.is-white .navbar-start .navbar-link.is-active,.navbar.is-white .navbar-start .navbar-link:focus,.navbar.is-white .navbar-start .navbar-link:hover,.navbar.is-white .navbar-start>a.navbar-item.is-active,.navbar.is-white .navbar-start>a.navbar-item:focus,.navbar.is-white .navbar-start>a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-end .navbar-link::after,.navbar.is-white .navbar-start .navbar-link::after{border-color:#0a0a0a}.navbar.is-white .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-white .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-white .navbar-item.has-dropdown:hover .navbar-link{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-dropdown a.navbar-item.is-active{background-color:#fff;color:#0a0a0a}}.navbar.is-black{background-color:#0a0a0a;color:#fff}.navbar.is-black .navbar-brand .navbar-link,.navbar.is-black .navbar-brand>.navbar-item{color:#fff}.navbar.is-black .navbar-brand .navbar-link.is-active,.navbar.is-black .navbar-brand .navbar-link:focus,.navbar.is-black .navbar-brand .navbar-link:hover,.navbar.is-black .navbar-brand>a.navbar-item.is-active,.navbar.is-black .navbar-brand>a.navbar-item:focus,.navbar.is-black .navbar-brand>a.navbar-item:hover{background-color:#000;color:#fff}.navbar.is-black .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-black .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-black .navbar-end .navbar-link,.navbar.is-black .navbar-end>.navbar-item,.navbar.is-black .navbar-start .navbar-link,.navbar.is-black .navbar-start>.navbar-item{color:#fff}.navbar.is-black .navbar-end .navbar-link.is-active,.navbar.is-black .navbar-end .navbar-link:focus,.navbar.is-black .navbar-end .navbar-link:hover,.navbar.is-black .navbar-end>a.navbar-item.is-active,.navbar.is-black .navbar-end>a.navbar-item:focus,.navbar.is-black .navbar-end>a.navbar-item:hover,.navbar.is-black .navbar-start .navbar-link.is-active,.navbar.is-black .navbar-start .navbar-link:focus,.navbar.is-black .navbar-start .navbar-link:hover,.navbar.is-black .navbar-start>a.navbar-item.is-active,.navbar.is-black .navbar-start>a.navbar-item:focus,.navbar.is-black .navbar-start>a.navbar-item:hover{background-color:#000;color:#fff}.navbar.is-black .navbar-end .navbar-link::after,.navbar.is-black .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-black .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-black .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-black .navbar-item.has-dropdown:hover .navbar-link{background-color:#000;color:#fff}.navbar.is-black .navbar-dropdown a.navbar-item.is-active{background-color:#0a0a0a;color:#fff}}.navbar.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link,.navbar.is-light .navbar-brand>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link.is-active,.navbar.is-light .navbar-brand .navbar-link:focus,.navbar.is-light .navbar-brand .navbar-link:hover,.navbar.is-light .navbar-brand>a.navbar-item.is-active,.navbar.is-light .navbar-brand>a.navbar-item:focus,.navbar.is-light .navbar-brand>a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-light .navbar-burger{color:rgba(0,0,0,.7)}@media screen and (min-width:1024px){.navbar.is-light .navbar-end .navbar-link,.navbar.is-light .navbar-end>.navbar-item,.navbar.is-light .navbar-start .navbar-link,.navbar.is-light .navbar-start>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-light .navbar-end .navbar-link.is-active,.navbar.is-light .navbar-end .navbar-link:focus,.navbar.is-light .navbar-end .navbar-link:hover,.navbar.is-light .navbar-end>a.navbar-item.is-active,.navbar.is-light .navbar-end>a.navbar-item:focus,.navbar.is-light .navbar-end>a.navbar-item:hover,.navbar.is-light .navbar-start .navbar-link.is-active,.navbar.is-light .navbar-start .navbar-link:focus,.navbar.is-light .navbar-start .navbar-link:hover,.navbar.is-light .navbar-start>a.navbar-item.is-active,.navbar.is-light .navbar-start>a.navbar-item:focus,.navbar.is-light .navbar-start>a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-end .navbar-link::after,.navbar.is-light .navbar-start .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-light .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-light .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-light .navbar-item.has-dropdown:hover .navbar-link{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:rgba(0,0,0,.7)}}.navbar.is-dark{background-color:#363636;color:#fff}.navbar.is-dark .navbar-brand .navbar-link,.navbar.is-dark .navbar-brand>.navbar-item{color:#fff}.navbar.is-dark .navbar-brand .navbar-link.is-active,.navbar.is-dark .navbar-brand .navbar-link:focus,.navbar.is-dark .navbar-brand .navbar-link:hover,.navbar.is-dark .navbar-brand>a.navbar-item.is-active,.navbar.is-dark .navbar-brand>a.navbar-item:focus,.navbar.is-dark .navbar-brand>a.navbar-item:hover{background-color:#292929;color:#fff}.navbar.is-dark .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-dark .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-dark .navbar-end .navbar-link,.navbar.is-dark .navbar-end>.navbar-item,.navbar.is-dark .navbar-start .navbar-link,.navbar.is-dark .navbar-start>.navbar-item{color:#fff}.navbar.is-dark .navbar-end .navbar-link.is-active,.navbar.is-dark .navbar-end .navbar-link:focus,.navbar.is-dark .navbar-end .navbar-link:hover,.navbar.is-dark .navbar-end>a.navbar-item.is-active,.navbar.is-dark .navbar-end>a.navbar-item:focus,.navbar.is-dark .navbar-end>a.navbar-item:hover,.navbar.is-dark .navbar-start .navbar-link.is-active,.navbar.is-dark .navbar-start .navbar-link:focus,.navbar.is-dark .navbar-start .navbar-link:hover,.navbar.is-dark .navbar-start>a.navbar-item.is-active,.navbar.is-dark .navbar-start>a.navbar-item:focus,.navbar.is-dark .navbar-start>a.navbar-item:hover{background-color:#292929;color:#fff}.navbar.is-dark .navbar-end .navbar-link::after,.navbar.is-dark .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-dark .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-dark .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-dark .navbar-item.has-dropdown:hover .navbar-link{background-color:#292929;color:#fff}.navbar.is-dark .navbar-dropdown a.navbar-item.is-active{background-color:#363636;color:#fff}}.navbar.is-primary{background-color:#00d1b2;color:#fff}.navbar.is-primary .navbar-brand .navbar-link,.navbar.is-primary .navbar-brand>.navbar-item{color:#fff}.navbar.is-primary .navbar-brand .navbar-link.is-active,.navbar.is-primary .navbar-brand .navbar-link:focus,.navbar.is-primary .navbar-brand .navbar-link:hover,.navbar.is-primary .navbar-brand>a.navbar-item.is-active,.navbar.is-primary .navbar-brand>a.navbar-item:focus,.navbar.is-primary .navbar-brand>a.navbar-item:hover{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-primary .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-primary .navbar-end .navbar-link,.navbar.is-primary .navbar-end>.navbar-item,.navbar.is-primary .navbar-start .navbar-link,.navbar.is-primary .navbar-start>.navbar-item{color:#fff}.navbar.is-primary .navbar-end .navbar-link.is-active,.navbar.is-primary .navbar-end .navbar-link:focus,.navbar.is-primary .navbar-end .navbar-link:hover,.navbar.is-primary .navbar-end>a.navbar-item.is-active,.navbar.is-primary .navbar-end>a.navbar-item:focus,.navbar.is-primary .navbar-end>a.navbar-item:hover,.navbar.is-primary .navbar-start .navbar-link.is-active,.navbar.is-primary .navbar-start .navbar-link:focus,.navbar.is-primary .navbar-start .navbar-link:hover,.navbar.is-primary .navbar-start>a.navbar-item.is-active,.navbar.is-primary .navbar-start>a.navbar-item:focus,.navbar.is-primary .navbar-start>a.navbar-item:hover{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-end .navbar-link::after,.navbar.is-primary .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-primary .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-primary .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-primary .navbar-item.has-dropdown:hover .navbar-link{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-dropdown a.navbar-item.is-active{background-color:#00d1b2;color:#fff}}.navbar.is-link{background-color:#3273dc;color:#fff}.navbar.is-link .navbar-brand .navbar-link,.navbar.is-link .navbar-brand>.navbar-item{color:#fff}.navbar.is-link .navbar-brand .navbar-link.is-active,.navbar.is-link .navbar-brand .navbar-link:focus,.navbar.is-link .navbar-brand .navbar-link:hover,.navbar.is-link .navbar-brand>a.navbar-item.is-active,.navbar.is-link .navbar-brand>a.navbar-item:focus,.navbar.is-link .navbar-brand>a.navbar-item:hover{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-link .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-link .navbar-end .navbar-link,.navbar.is-link .navbar-end>.navbar-item,.navbar.is-link .navbar-start .navbar-link,.navbar.is-link .navbar-start>.navbar-item{color:#fff}.navbar.is-link .navbar-end .navbar-link.is-active,.navbar.is-link .navbar-end .navbar-link:focus,.navbar.is-link .navbar-end .navbar-link:hover,.navbar.is-link .navbar-end>a.navbar-item.is-active,.navbar.is-link .navbar-end>a.navbar-item:focus,.navbar.is-link .navbar-end>a.navbar-item:hover,.navbar.is-link .navbar-start .navbar-link.is-active,.navbar.is-link .navbar-start .navbar-link:focus,.navbar.is-link .navbar-start .navbar-link:hover,.navbar.is-link .navbar-start>a.navbar-item.is-active,.navbar.is-link .navbar-start>a.navbar-item:focus,.navbar.is-link .navbar-start>a.navbar-item:hover{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-end .navbar-link::after,.navbar.is-link .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-link .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-link .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-link .navbar-item.has-dropdown:hover .navbar-link{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-dropdown a.navbar-item.is-active{background-color:#3273dc;color:#fff}}.navbar.is-info{background-color:#3298dc;color:#fff}.navbar.is-info .navbar-brand .navbar-link,.navbar.is-info .navbar-brand>.navbar-item{color:#fff}.navbar.is-info .navbar-brand .navbar-link.is-active,.navbar.is-info .navbar-brand .navbar-link:focus,.navbar.is-info .navbar-brand .navbar-link:hover,.navbar.is-info .navbar-brand>a.navbar-item.is-active,.navbar.is-info .navbar-brand>a.navbar-item:focus,.navbar.is-info .navbar-brand>a.navbar-item:hover{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-info .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-info .navbar-end .navbar-link,.navbar.is-info .navbar-end>.navbar-item,.navbar.is-info .navbar-start .navbar-link,.navbar.is-info .navbar-start>.navbar-item{color:#fff}.navbar.is-info .navbar-end .navbar-link.is-active,.navbar.is-info .navbar-end .navbar-link:focus,.navbar.is-info .navbar-end .navbar-link:hover,.navbar.is-info .navbar-end>a.navbar-item.is-active,.navbar.is-info .navbar-end>a.navbar-item:focus,.navbar.is-info .navbar-end>a.navbar-item:hover,.navbar.is-info .navbar-start .navbar-link.is-active,.navbar.is-info .navbar-start .navbar-link:focus,.navbar.is-info .navbar-start .navbar-link:hover,.navbar.is-info .navbar-start>a.navbar-item.is-active,.navbar.is-info .navbar-start>a.navbar-item:focus,.navbar.is-info .navbar-start>a.navbar-item:hover{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-end .navbar-link::after,.navbar.is-info .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-info .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-info .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-info .navbar-item.has-dropdown:hover .navbar-link{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-dropdown a.navbar-item.is-active{background-color:#3298dc;color:#fff}}.navbar.is-success{background-color:#48c774;color:#fff}.navbar.is-success .navbar-brand .navbar-link,.navbar.is-success .navbar-brand>.navbar-item{color:#fff}.navbar.is-success .navbar-brand .navbar-link.is-active,.navbar.is-success .navbar-brand .navbar-link:focus,.navbar.is-success .navbar-brand .navbar-link:hover,.navbar.is-success .navbar-brand>a.navbar-item.is-active,.navbar.is-success .navbar-brand>a.navbar-item:focus,.navbar.is-success .navbar-brand>a.navbar-item:hover{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-success .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-success .navbar-end .navbar-link,.navbar.is-success .navbar-end>.navbar-item,.navbar.is-success .navbar-start .navbar-link,.navbar.is-success .navbar-start>.navbar-item{color:#fff}.navbar.is-success .navbar-end .navbar-link.is-active,.navbar.is-success .navbar-end .navbar-link:focus,.navbar.is-success .navbar-end .navbar-link:hover,.navbar.is-success .navbar-end>a.navbar-item.is-active,.navbar.is-success .navbar-end>a.navbar-item:focus,.navbar.is-success .navbar-end>a.navbar-item:hover,.navbar.is-success .navbar-start .navbar-link.is-active,.navbar.is-success .navbar-start .navbar-link:focus,.navbar.is-success .navbar-start .navbar-link:hover,.navbar.is-success .navbar-start>a.navbar-item.is-active,.navbar.is-success .navbar-start>a.navbar-item:focus,.navbar.is-success .navbar-start>a.navbar-item:hover{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-end .navbar-link::after,.navbar.is-success .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-success .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-success .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-success .navbar-item.has-dropdown:hover .navbar-link{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-dropdown a.navbar-item.is-active{background-color:#48c774;color:#fff}}.navbar.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link,.navbar.is-warning .navbar-brand>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link.is-active,.navbar.is-warning .navbar-brand .navbar-link:focus,.navbar.is-warning .navbar-brand .navbar-link:hover,.navbar.is-warning .navbar-brand>a.navbar-item.is-active,.navbar.is-warning .navbar-brand>a.navbar-item:focus,.navbar.is-warning .navbar-brand>a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-burger{color:rgba(0,0,0,.7)}@media screen and (min-width:1024px){.navbar.is-warning .navbar-end .navbar-link,.navbar.is-warning .navbar-end>.navbar-item,.navbar.is-warning .navbar-start .navbar-link,.navbar.is-warning .navbar-start>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-end .navbar-link.is-active,.navbar.is-warning .navbar-end .navbar-link:focus,.navbar.is-warning .navbar-end .navbar-link:hover,.navbar.is-warning .navbar-end>a.navbar-item.is-active,.navbar.is-warning .navbar-end>a.navbar-item:focus,.navbar.is-warning .navbar-end>a.navbar-item:hover,.navbar.is-warning .navbar-start .navbar-link.is-active,.navbar.is-warning .navbar-start .navbar-link:focus,.navbar.is-warning .navbar-start .navbar-link:hover,.navbar.is-warning .navbar-start>a.navbar-item.is-active,.navbar.is-warning .navbar-start>a.navbar-item:focus,.navbar.is-warning .navbar-start>a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-end .navbar-link::after,.navbar.is-warning .navbar-start .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-warning .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-warning .navbar-item.has-dropdown:hover .navbar-link{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-dropdown a.navbar-item.is-active{background-color:#ffdd57;color:rgba(0,0,0,.7)}}.navbar.is-danger{background-color:#f14668;color:#fff}.navbar.is-danger .navbar-brand .navbar-link,.navbar.is-danger .navbar-brand>.navbar-item{color:#fff}.navbar.is-danger .navbar-brand .navbar-link.is-active,.navbar.is-danger .navbar-brand .navbar-link:focus,.navbar.is-danger .navbar-brand .navbar-link:hover,.navbar.is-danger .navbar-brand>a.navbar-item.is-active,.navbar.is-danger .navbar-brand>a.navbar-item:focus,.navbar.is-danger .navbar-brand>a.navbar-item:hover{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-danger .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-danger .navbar-end .navbar-link,.navbar.is-danger .navbar-end>.navbar-item,.navbar.is-danger .navbar-start .navbar-link,.navbar.is-danger .navbar-start>.navbar-item{color:#fff}.navbar.is-danger .navbar-end .navbar-link.is-active,.navbar.is-danger .navbar-end .navbar-link:focus,.navbar.is-danger .navbar-end .navbar-link:hover,.navbar.is-danger .navbar-end>a.navbar-item.is-active,.navbar.is-danger .navbar-end>a.navbar-item:focus,.navbar.is-danger .navbar-end>a.navbar-item:hover,.navbar.is-danger .navbar-start .navbar-link.is-active,.navbar.is-danger .navbar-start .navbar-link:focus,.navbar.is-danger .navbar-start .navbar-link:hover,.navbar.is-danger .navbar-start>a.navbar-item.is-active,.navbar.is-danger .navbar-start>a.navbar-item:focus,.navbar.is-danger .navbar-start>a.navbar-item:hover{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-end .navbar-link::after,.navbar.is-danger .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-danger .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-danger .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-danger .navbar-item.has-dropdown:hover .navbar-link{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-dropdown a.navbar-item.is-active{background-color:#f14668;color:#fff}}.navbar>.container{align-items:stretch;display:flex;min-height:3.25rem;width:100%}.navbar.has-shadow{box-shadow:0 2px 0 0 #f5f5f5}.navbar.is-fixed-bottom,.navbar.is-fixed-top{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom{bottom:0}.navbar.is-fixed-bottom.has-shadow{box-shadow:0 -2px 0 0 #f5f5f5}.navbar.is-fixed-top{top:0}body.has-navbar-fixed-top,html.has-navbar-fixed-top{padding-top:3.25rem}body.has-navbar-fixed-bottom,html.has-navbar-fixed-bottom{padding-bottom:3.25rem}.navbar-brand,.navbar-tabs{align-items:stretch;display:flex;flex-shrink:0;min-height:3.25rem}.navbar-brand a.navbar-item:focus,.navbar-brand a.navbar-item:hover{background-color:transparent}.navbar-tabs{-webkit-overflow-scrolling:touch;max-width:100vw;overflow-x:auto;overflow-y:hidden}.navbar-burger{color:#4a4a4a;cursor:pointer;display:block;height:3.25rem;position:relative;width:3.25rem;margin-left:auto}.navbar-burger span{background-color:currentColor;display:block;height:1px;left:calc(50% - 8px);position:absolute;transform-origin:center;transition-duration:86ms;transition-property:background-color,opacity,transform;transition-timing-function:ease-out;width:16px}.navbar-burger span:nth-child(1){top:calc(50% - 6px)}.navbar-burger span:nth-child(2){top:calc(50% - 1px)}.navbar-burger span:nth-child(3){top:calc(50% + 4px)}.navbar-burger:hover{background-color:rgba(0,0,0,.05)}.navbar-burger.is-active span:nth-child(1){transform:translateY(5px) rotate(45deg)}.navbar-burger.is-active span:nth-child(2){opacity:0}.navbar-burger.is-active span:nth-child(3){transform:translateY(-5px) rotate(-45deg)}.navbar-menu{display:none}.navbar-item,.navbar-link{color:#4a4a4a;display:block;line-height:1.5;padding:.5rem .75rem;position:relative}.navbar-item .icon:only-child,.navbar-link .icon:only-child{margin-left:-.25rem;margin-right:-.25rem}.navbar-link,a.navbar-item{cursor:pointer}.navbar-link.is-active,.navbar-link:focus,.navbar-link:focus-within,.navbar-link:hover,a.navbar-item.is-active,a.navbar-item:focus,a.navbar-item:focus-within,a.navbar-item:hover{background-color:#fafafa;color:#3273dc}.navbar-item{display:block;flex-grow:0;flex-shrink:0}.navbar-item img{max-height:1.75rem}.navbar-item.has-dropdown{padding:0}.navbar-item.is-expanded{flex-grow:1;flex-shrink:1}.navbar-item.is-tab{border-bottom:1px solid transparent;min-height:3.25rem;padding-bottom:calc(.5rem - 1px)}.navbar-item.is-tab:focus,.navbar-item.is-tab:hover{background-color:transparent;border-bottom-color:#3273dc}.navbar-item.is-tab.is-active{background-color:transparent;border-bottom-color:#3273dc;border-bottom-style:solid;border-bottom-width:3px;color:#3273dc;padding-bottom:calc(.5rem - 3px)}.navbar-content{flex-grow:1;flex-shrink:1}.navbar-link:not(.is-arrowless){padding-right:2.5em}.navbar-link:not(.is-arrowless)::after{border-color:#3273dc;margin-top:-.375em;right:1.125em}.navbar-dropdown{font-size:.875rem;padding-bottom:.5rem;padding-top:.5rem}.navbar-dropdown .navbar-item{padding-left:1.5rem;padding-right:1.5rem}.navbar-divider{background-color:#f5f5f5;border:none;display:none;height:2px;margin:.5rem 0}@media screen and (max-width:1023px){.navbar>.container{display:block}.navbar-brand .navbar-item,.navbar-tabs .navbar-item{align-items:center;display:flex}.navbar-link::after{display:none}.navbar-menu{background-color:#fff;box-shadow:0 8px 16px rgba(10,10,10,.1);padding:.5rem 0}.navbar-menu.is-active{display:block}.navbar.is-fixed-bottom-touch,.navbar.is-fixed-top-touch{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom-touch{bottom:0}.navbar.is-fixed-bottom-touch.has-shadow{box-shadow:0 -2px 3px rgba(10,10,10,.1)}.navbar.is-fixed-top-touch{top:0}.navbar.is-fixed-top .navbar-menu,.navbar.is-fixed-top-touch .navbar-menu{-webkit-overflow-scrolling:touch;max-height:calc(100vh - 3.25rem);overflow:auto}body.has-navbar-fixed-top-touch,html.has-navbar-fixed-top-touch{padding-top:3.25rem}body.has-navbar-fixed-bottom-touch,html.has-navbar-fixed-bottom-touch{padding-bottom:3.25rem}}@media screen and (min-width:1024px){.navbar,.navbar-end,.navbar-menu,.navbar-start{align-items:stretch;display:flex}.navbar{min-height:3.25rem}.navbar.is-spaced{padding:1rem 2rem}.navbar.is-spaced .navbar-end,.navbar.is-spaced .navbar-start{align-items:center}.navbar.is-spaced .navbar-link,.navbar.is-spaced a.navbar-item{border-radius:4px}.navbar.is-transparent .navbar-link.is-active,.navbar.is-transparent .navbar-link:focus,.navbar.is-transparent .navbar-link:hover,.navbar.is-transparent a.navbar-item.is-active,.navbar.is-transparent a.navbar-item:focus,.navbar.is-transparent a.navbar-item:hover{background-color:transparent!important}.navbar.is-transparent .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:focus .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:focus-within .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:hover .navbar-link{background-color:transparent!important}.navbar.is-transparent .navbar-dropdown a.navbar-item:focus,.navbar.is-transparent .navbar-dropdown a.navbar-item:hover{background-color:#f5f5f5;color:#0a0a0a}.navbar.is-transparent .navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:#3273dc}.navbar-burger{display:none}.navbar-item,.navbar-link{align-items:center;display:flex}.navbar-item{display:flex}.navbar-item.has-dropdown{align-items:stretch}.navbar-item.has-dropdown-up .navbar-link::after{transform:rotate(135deg) translate(.25em,-.25em)}.navbar-item.has-dropdown-up .navbar-dropdown{border-bottom:2px solid #dbdbdb;border-radius:6px 6px 0 0;border-top:none;bottom:100%;box-shadow:0 -8px 8px rgba(10,10,10,.1);top:auto}.navbar-item.is-active .navbar-dropdown,.navbar-item.is-hoverable:focus .navbar-dropdown,.navbar-item.is-hoverable:focus-within .navbar-dropdown,.navbar-item.is-hoverable:hover .navbar-dropdown{display:block}.navbar-item.is-active .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:focus .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:focus-within .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:hover .navbar-dropdown.is-boxed,.navbar.is-spaced .navbar-item.is-active .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:focus .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:focus-within .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:hover .navbar-dropdown{opacity:1;pointer-events:auto;transform:translateY(0)}.navbar-menu{flex-grow:1;flex-shrink:0}.navbar-start{justify-content:flex-start;margin-right:auto}.navbar-end{justify-content:flex-end;margin-left:auto}.navbar-dropdown{background-color:#fff;border-bottom-left-radius:6px;border-bottom-right-radius:6px;border-top:2px solid #dbdbdb;box-shadow:0 8px 8px rgba(10,10,10,.1);display:none;font-size:.875rem;left:0;min-width:100%;position:absolute;top:100%;z-index:20}.navbar-dropdown .navbar-item{padding:.375rem 1rem;white-space:nowrap}.navbar-dropdown a.navbar-item{padding-right:3rem}.navbar-dropdown a.navbar-item:focus,.navbar-dropdown a.navbar-item:hover{background-color:#f5f5f5;color:#0a0a0a}.navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:#3273dc}.navbar-dropdown.is-boxed,.navbar.is-spaced .navbar-dropdown{border-radius:6px;border-top:none;box-shadow:0 8px 8px rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.1);display:block;opacity:0;pointer-events:none;top:calc(100% + (-4px));transform:translateY(-5px);transition-duration:86ms;transition-property:opacity,transform}.navbar-dropdown.is-right{left:auto;right:0}.navbar-divider{display:block}.container>.navbar .navbar-brand,.navbar>.container .navbar-brand{margin-left:-.75rem}.container>.navbar .navbar-menu,.navbar>.container .navbar-menu{margin-right:-.75rem}.navbar.is-fixed-bottom-desktop,.navbar.is-fixed-top-desktop{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom-desktop{bottom:0}.navbar.is-fixed-bottom-desktop.has-shadow{box-shadow:0 -2px 3px rgba(10,10,10,.1)}.navbar.is-fixed-top-desktop{top:0}body.has-navbar-fixed-top-desktop,html.has-navbar-fixed-top-desktop{padding-top:3.25rem}body.has-navbar-fixed-bottom-desktop,html.has-navbar-fixed-bottom-desktop{padding-bottom:3.25rem}body.has-spaced-navbar-fixed-top,html.has-spaced-navbar-fixed-top{padding-top:5.25rem}body.has-spaced-navbar-fixed-bottom,html.has-spaced-navbar-fixed-bottom{padding-bottom:5.25rem}.navbar-link.is-active,a.navbar-item.is-active{color:#0a0a0a}.navbar-link.is-active:not(:focus):not(:hover),a.navbar-item.is-active:not(:focus):not(:hover){background-color:transparent}.navbar-item.has-dropdown.is-active .navbar-link,.navbar-item.has-dropdown:focus .navbar-link,.navbar-item.has-dropdown:hover .navbar-link{background-color:#fafafa}}.hero.is-fullheight-with-navbar{min-height:calc(100vh - 3.25rem)}.pagination{font-size:1rem;margin:-.25rem}.pagination.is-small{font-size:.75rem}.pagination.is-medium{font-size:1.25rem}.pagination.is-large{font-size:1.5rem}.pagination.is-rounded .pagination-next,.pagination.is-rounded .pagination-previous{padding-left:1em;padding-right:1em;border-radius:290486px}.pagination.is-rounded .pagination-link{border-radius:290486px}.pagination,.pagination-list{align-items:center;display:flex;justify-content:center;text-align:center}.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous{font-size:1em;justify-content:center;margin:.25rem;padding-left:.5em;padding-right:.5em;text-align:center}.pagination-link,.pagination-next,.pagination-previous{border-color:#dbdbdb;color:#363636;min-width:2.5em}.pagination-link:hover,.pagination-next:hover,.pagination-previous:hover{border-color:#b5b5b5;color:#363636}.pagination-link:focus,.pagination-next:focus,.pagination-previous:focus{border-color:#3273dc}.pagination-link:active,.pagination-next:active,.pagination-previous:active{box-shadow:inset 0 1px 2px rgba(10,10,10,.2)}.pagination-link[disabled],.pagination-next[disabled],.pagination-previous[disabled]{background-color:#dbdbdb;border-color:#dbdbdb;box-shadow:none;color:#7a7a7a;opacity:.5}.pagination-next,.pagination-previous{padding-left:.75em;padding-right:.75em;white-space:nowrap}.pagination-link.is-current{background-color:#3273dc;border-color:#3273dc;color:#fff}.pagination-ellipsis{color:#b5b5b5;pointer-events:none}.pagination-list{flex-wrap:wrap}@media screen and (max-width:768px){.pagination{flex-wrap:wrap}.pagination-next,.pagination-previous{flex-grow:1;flex-shrink:1}.pagination-list li{flex-grow:1;flex-shrink:1}}@media screen and (min-width:769px),print{.pagination-list{flex-grow:1;flex-shrink:1;justify-content:flex-start;order:1}.pagination-previous{order:2}.pagination-next{order:3}.pagination{justify-content:space-between}.pagination.is-centered .pagination-previous{order:1}.pagination.is-centered .pagination-list{justify-content:center;order:2}.pagination.is-centered .pagination-next{order:3}.pagination.is-right .pagination-previous{order:1}.pagination.is-right .pagination-next{order:2}.pagination.is-right .pagination-list{justify-content:flex-end;order:3}}.panel{border-radius:6px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);font-size:1rem}.panel:not(:last-child){margin-bottom:1.5rem}.panel.is-white .panel-heading{background-color:#fff;color:#0a0a0a}.panel.is-white .panel-tabs a.is-active{border-bottom-color:#fff}.panel.is-white .panel-block.is-active .panel-icon{color:#fff}.panel.is-black .panel-heading{background-color:#0a0a0a;color:#fff}.panel.is-black .panel-tabs a.is-active{border-bottom-color:#0a0a0a}.panel.is-black .panel-block.is-active .panel-icon{color:#0a0a0a}.panel.is-light .panel-heading{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.panel.is-light .panel-tabs a.is-active{border-bottom-color:#f5f5f5}.panel.is-light .panel-block.is-active .panel-icon{color:#f5f5f5}.panel.is-dark .panel-heading{background-color:#363636;color:#fff}.panel.is-dark .panel-tabs a.is-active{border-bottom-color:#363636}.panel.is-dark .panel-block.is-active .panel-icon{color:#363636}.panel.is-primary .panel-heading{background-color:#00d1b2;color:#fff}.panel.is-primary .panel-tabs a.is-active{border-bottom-color:#00d1b2}.panel.is-primary .panel-block.is-active .panel-icon{color:#00d1b2}.panel.is-link .panel-heading{background-color:#3273dc;color:#fff}.panel.is-link .panel-tabs a.is-active{border-bottom-color:#3273dc}.panel.is-link .panel-block.is-active .panel-icon{color:#3273dc}.panel.is-info .panel-heading{background-color:#3298dc;color:#fff}.panel.is-info .panel-tabs a.is-active{border-bottom-color:#3298dc}.panel.is-info .panel-block.is-active .panel-icon{color:#3298dc}.panel.is-success .panel-heading{background-color:#48c774;color:#fff}.panel.is-success .panel-tabs a.is-active{border-bottom-color:#48c774}.panel.is-success .panel-block.is-active .panel-icon{color:#48c774}.panel.is-warning .panel-heading{background-color:#ffdd57;color:rgba(0,0,0,.7)}.panel.is-warning .panel-tabs a.is-active{border-bottom-color:#ffdd57}.panel.is-warning .panel-block.is-active .panel-icon{color:#ffdd57}.panel.is-danger .panel-heading{background-color:#f14668;color:#fff}.panel.is-danger .panel-tabs a.is-active{border-bottom-color:#f14668}.panel.is-danger .panel-block.is-active .panel-icon{color:#f14668}.panel-block:not(:last-child),.panel-tabs:not(:last-child){border-bottom:1px solid #ededed}.panel-heading{background-color:#ededed;border-radius:6px 6px 0 0;color:#363636;font-size:1.25em;font-weight:700;line-height:1.25;padding:.75em 1em}.panel-tabs{align-items:flex-end;display:flex;font-size:.875em;justify-content:center}.panel-tabs a{border-bottom:1px solid #dbdbdb;margin-bottom:-1px;padding:.5em}.panel-tabs a.is-active{border-bottom-color:#4a4a4a;color:#363636}.panel-list a{color:#4a4a4a}.panel-list a:hover{color:#3273dc}.panel-block{align-items:center;color:#363636;display:flex;justify-content:flex-start;padding:.5em .75em}.panel-block input[type=checkbox]{margin-right:.75em}.panel-block>.control{flex-grow:1;flex-shrink:1;width:100%}.panel-block.is-wrapped{flex-wrap:wrap}.panel-block.is-active{border-left-color:#3273dc;color:#363636}.panel-block.is-active .panel-icon{color:#3273dc}.panel-block:last-child{border-bottom-left-radius:6px;border-bottom-right-radius:6px}a.panel-block,label.panel-block{cursor:pointer}a.panel-block:hover,label.panel-block:hover{background-color:#f5f5f5}.panel-icon{display:inline-block;font-size:14px;height:1em;line-height:1em;text-align:center;vertical-align:top;width:1em;color:#7a7a7a;margin-right:.75em}.panel-icon .fa{font-size:inherit;line-height:inherit}.tabs{-webkit-overflow-scrolling:touch;align-items:stretch;display:flex;font-size:1rem;justify-content:space-between;overflow:hidden;overflow-x:auto;white-space:nowrap}.tabs a{align-items:center;border-bottom-color:#dbdbdb;border-bottom-style:solid;border-bottom-width:1px;color:#4a4a4a;display:flex;justify-content:center;margin-bottom:-1px;padding:.5em 1em;vertical-align:top}.tabs a:hover{border-bottom-color:#363636;color:#363636}.tabs li{display:block}.tabs li.is-active a{border-bottom-color:#3273dc;color:#3273dc}.tabs ul{align-items:center;border-bottom-color:#dbdbdb;border-bottom-style:solid;border-bottom-width:1px;display:flex;flex-grow:1;flex-shrink:0;justify-content:flex-start}.tabs ul.is-left{padding-right:.75em}.tabs ul.is-center{flex:none;justify-content:center;padding-left:.75em;padding-right:.75em}.tabs ul.is-right{justify-content:flex-end;padding-left:.75em}.tabs .icon:first-child{margin-right:.5em}.tabs .icon:last-child{margin-left:.5em}.tabs.is-centered ul{justify-content:center}.tabs.is-right ul{justify-content:flex-end}.tabs.is-boxed a{border:1px solid transparent;border-radius:4px 4px 0 0}.tabs.is-boxed a:hover{background-color:#f5f5f5;border-bottom-color:#dbdbdb}.tabs.is-boxed li.is-active a{background-color:#fff;border-color:#dbdbdb;border-bottom-color:transparent!important}.tabs.is-fullwidth li{flex-grow:1;flex-shrink:0}.tabs.is-toggle a{border-color:#dbdbdb;border-style:solid;border-width:1px;margin-bottom:0;position:relative}.tabs.is-toggle a:hover{background-color:#f5f5f5;border-color:#b5b5b5;z-index:2}.tabs.is-toggle li+li{margin-left:-1px}.tabs.is-toggle li:first-child a{border-radius:4px 0 0 4px}.tabs.is-toggle li:last-child a{border-radius:0 4px 4px 0}.tabs.is-toggle li.is-active a{background-color:#3273dc;border-color:#3273dc;color:#fff;z-index:1}.tabs.is-toggle ul{border-bottom:none}.tabs.is-toggle.is-toggle-rounded li:first-child a{border-bottom-left-radius:290486px;border-top-left-radius:290486px;padding-left:1.25em}.tabs.is-toggle.is-toggle-rounded li:last-child a{border-bottom-right-radius:290486px;border-top-right-radius:290486px;padding-right:1.25em}.tabs.is-small{font-size:.75rem}.tabs.is-medium{font-size:1.25rem}.tabs.is-large{font-size:1.5rem}.column{display:block;flex-basis:0;flex-grow:1;flex-shrink:1;padding:.75rem}.columns.is-mobile>.column.is-narrow{flex:none}.columns.is-mobile>.column.is-full{flex:none;width:100%}.columns.is-mobile>.column.is-three-quarters{flex:none;width:75%}.columns.is-mobile>.column.is-two-thirds{flex:none;width:66.6666%}.columns.is-mobile>.column.is-half{flex:none;width:50%}.columns.is-mobile>.column.is-one-third{flex:none;width:33.3333%}.columns.is-mobile>.column.is-one-quarter{flex:none;width:25%}.columns.is-mobile>.column.is-one-fifth{flex:none;width:20%}.columns.is-mobile>.column.is-two-fifths{flex:none;width:40%}.columns.is-mobile>.column.is-three-fifths{flex:none;width:60%}.columns.is-mobile>.column.is-four-fifths{flex:none;width:80%}.columns.is-mobile>.column.is-offset-three-quarters{margin-left:75%}.columns.is-mobile>.column.is-offset-two-thirds{margin-left:66.6666%}.columns.is-mobile>.column.is-offset-half{margin-left:50%}.columns.is-mobile>.column.is-offset-one-third{margin-left:33.3333%}.columns.is-mobile>.column.is-offset-one-quarter{margin-left:25%}.columns.is-mobile>.column.is-offset-one-fifth{margin-left:20%}.columns.is-mobile>.column.is-offset-two-fifths{margin-left:40%}.columns.is-mobile>.column.is-offset-three-fifths{margin-left:60%}.columns.is-mobile>.column.is-offset-four-fifths{margin-left:80%}.columns.is-mobile>.column.is-0{flex:none;width:0%}.columns.is-mobile>.column.is-offset-0{margin-left:0}.columns.is-mobile>.column.is-1{flex:none;width:8.33333%}.columns.is-mobile>.column.is-offset-1{margin-left:8.33333%}.columns.is-mobile>.column.is-2{flex:none;width:16.66667%}.columns.is-mobile>.column.is-offset-2{margin-left:16.66667%}.columns.is-mobile>.column.is-3{flex:none;width:25%}.columns.is-mobile>.column.is-offset-3{margin-left:25%}.columns.is-mobile>.column.is-4{flex:none;width:33.33333%}.columns.is-mobile>.column.is-offset-4{margin-left:33.33333%}.columns.is-mobile>.column.is-5{flex:none;width:41.66667%}.columns.is-mobile>.column.is-offset-5{margin-left:41.66667%}.columns.is-mobile>.column.is-6{flex:none;width:50%}.columns.is-mobile>.column.is-offset-6{margin-left:50%}.columns.is-mobile>.column.is-7{flex:none;width:58.33333%}.columns.is-mobile>.column.is-offset-7{margin-left:58.33333%}.columns.is-mobile>.column.is-8{flex:none;width:66.66667%}.columns.is-mobile>.column.is-offset-8{margin-left:66.66667%}.columns.is-mobile>.column.is-9{flex:none;width:75%}.columns.is-mobile>.column.is-offset-9{margin-left:75%}.columns.is-mobile>.column.is-10{flex:none;width:83.33333%}.columns.is-mobile>.column.is-offset-10{margin-left:83.33333%}.columns.is-mobile>.column.is-11{flex:none;width:91.66667%}.columns.is-mobile>.column.is-offset-11{margin-left:91.66667%}.columns.is-mobile>.column.is-12{flex:none;width:100%}.columns.is-mobile>.column.is-offset-12{margin-left:100%}@media screen and (max-width:768px){.column.is-narrow-mobile{flex:none}.column.is-full-mobile{flex:none;width:100%}.column.is-three-quarters-mobile{flex:none;width:75%}.column.is-two-thirds-mobile{flex:none;width:66.6666%}.column.is-half-mobile{flex:none;width:50%}.column.is-one-third-mobile{flex:none;width:33.3333%}.column.is-one-quarter-mobile{flex:none;width:25%}.column.is-one-fifth-mobile{flex:none;width:20%}.column.is-two-fifths-mobile{flex:none;width:40%}.column.is-three-fifths-mobile{flex:none;width:60%}.column.is-four-fifths-mobile{flex:none;width:80%}.column.is-offset-three-quarters-mobile{margin-left:75%}.column.is-offset-two-thirds-mobile{margin-left:66.6666%}.column.is-offset-half-mobile{margin-left:50%}.column.is-offset-one-third-mobile{margin-left:33.3333%}.column.is-offset-one-quarter-mobile{margin-left:25%}.column.is-offset-one-fifth-mobile{margin-left:20%}.column.is-offset-two-fifths-mobile{margin-left:40%}.column.is-offset-three-fifths-mobile{margin-left:60%}.column.is-offset-four-fifths-mobile{margin-left:80%}.column.is-0-mobile{flex:none;width:0%}.column.is-offset-0-mobile{margin-left:0}.column.is-1-mobile{flex:none;width:8.33333%}.column.is-offset-1-mobile{margin-left:8.33333%}.column.is-2-mobile{flex:none;width:16.66667%}.column.is-offset-2-mobile{margin-left:16.66667%}.column.is-3-mobile{flex:none;width:25%}.column.is-offset-3-mobile{margin-left:25%}.column.is-4-mobile{flex:none;width:33.33333%}.column.is-offset-4-mobile{margin-left:33.33333%}.column.is-5-mobile{flex:none;width:41.66667%}.column.is-offset-5-mobile{margin-left:41.66667%}.column.is-6-mobile{flex:none;width:50%}.column.is-offset-6-mobile{margin-left:50%}.column.is-7-mobile{flex:none;width:58.33333%}.column.is-offset-7-mobile{margin-left:58.33333%}.column.is-8-mobile{flex:none;width:66.66667%}.column.is-offset-8-mobile{margin-left:66.66667%}.column.is-9-mobile{flex:none;width:75%}.column.is-offset-9-mobile{margin-left:75%}.column.is-10-mobile{flex:none;width:83.33333%}.column.is-offset-10-mobile{margin-left:83.33333%}.column.is-11-mobile{flex:none;width:91.66667%}.column.is-offset-11-mobile{margin-left:91.66667%}.column.is-12-mobile{flex:none;width:100%}.column.is-offset-12-mobile{margin-left:100%}}@media screen and (min-width:769px),print{.column.is-narrow,.column.is-narrow-tablet{flex:none}.column.is-full,.column.is-full-tablet{flex:none;width:100%}.column.is-three-quarters,.column.is-three-quarters-tablet{flex:none;width:75%}.column.is-two-thirds,.column.is-two-thirds-tablet{flex:none;width:66.6666%}.column.is-half,.column.is-half-tablet{flex:none;width:50%}.column.is-one-third,.column.is-one-third-tablet{flex:none;width:33.3333%}.column.is-one-quarter,.column.is-one-quarter-tablet{flex:none;width:25%}.column.is-one-fifth,.column.is-one-fifth-tablet{flex:none;width:20%}.column.is-two-fifths,.column.is-two-fifths-tablet{flex:none;width:40%}.column.is-three-fifths,.column.is-three-fifths-tablet{flex:none;width:60%}.column.is-four-fifths,.column.is-four-fifths-tablet{flex:none;width:80%}.column.is-offset-three-quarters,.column.is-offset-three-quarters-tablet{margin-left:75%}.column.is-offset-two-thirds,.column.is-offset-two-thirds-tablet{margin-left:66.6666%}.column.is-offset-half,.column.is-offset-half-tablet{margin-left:50%}.column.is-offset-one-third,.column.is-offset-one-third-tablet{margin-left:33.3333%}.column.is-offset-one-quarter,.column.is-offset-one-quarter-tablet{margin-left:25%}.column.is-offset-one-fifth,.column.is-offset-one-fifth-tablet{margin-left:20%}.column.is-offset-two-fifths,.column.is-offset-two-fifths-tablet{margin-left:40%}.column.is-offset-three-fifths,.column.is-offset-three-fifths-tablet{margin-left:60%}.column.is-offset-four-fifths,.column.is-offset-four-fifths-tablet{margin-left:80%}.column.is-0,.column.is-0-tablet{flex:none;width:0%}.column.is-offset-0,.column.is-offset-0-tablet{margin-left:0}.column.is-1,.column.is-1-tablet{flex:none;width:8.33333%}.column.is-offset-1,.column.is-offset-1-tablet{margin-left:8.33333%}.column.is-2,.column.is-2-tablet{flex:none;width:16.66667%}.column.is-offset-2,.column.is-offset-2-tablet{margin-left:16.66667%}.column.is-3,.column.is-3-tablet{flex:none;width:25%}.column.is-offset-3,.column.is-offset-3-tablet{margin-left:25%}.column.is-4,.column.is-4-tablet{flex:none;width:33.33333%}.column.is-offset-4,.column.is-offset-4-tablet{margin-left:33.33333%}.column.is-5,.column.is-5-tablet{flex:none;width:41.66667%}.column.is-offset-5,.column.is-offset-5-tablet{margin-left:41.66667%}.column.is-6,.column.is-6-tablet{flex:none;width:50%}.column.is-offset-6,.column.is-offset-6-tablet{margin-left:50%}.column.is-7,.column.is-7-tablet{flex:none;width:58.33333%}.column.is-offset-7,.column.is-offset-7-tablet{margin-left:58.33333%}.column.is-8,.column.is-8-tablet{flex:none;width:66.66667%}.column.is-offset-8,.column.is-offset-8-tablet{margin-left:66.66667%}.column.is-9,.column.is-9-tablet{flex:none;width:75%}.column.is-offset-9,.column.is-offset-9-tablet{margin-left:75%}.column.is-10,.column.is-10-tablet{flex:none;width:83.33333%}.column.is-offset-10,.column.is-offset-10-tablet{margin-left:83.33333%}.column.is-11,.column.is-11-tablet{flex:none;width:91.66667%}.column.is-offset-11,.column.is-offset-11-tablet{margin-left:91.66667%}.column.is-12,.column.is-12-tablet{flex:none;width:100%}.column.is-offset-12,.column.is-offset-12-tablet{margin-left:100%}}@media screen and (max-width:1023px){.column.is-narrow-touch{flex:none}.column.is-full-touch{flex:none;width:100%}.column.is-three-quarters-touch{flex:none;width:75%}.column.is-two-thirds-touch{flex:none;width:66.6666%}.column.is-half-touch{flex:none;width:50%}.column.is-one-third-touch{flex:none;width:33.3333%}.column.is-one-quarter-touch{flex:none;width:25%}.column.is-one-fifth-touch{flex:none;width:20%}.column.is-two-fifths-touch{flex:none;width:40%}.column.is-three-fifths-touch{flex:none;width:60%}.column.is-four-fifths-touch{flex:none;width:80%}.column.is-offset-three-quarters-touch{margin-left:75%}.column.is-offset-two-thirds-touch{margin-left:66.6666%}.column.is-offset-half-touch{margin-left:50%}.column.is-offset-one-third-touch{margin-left:33.3333%}.column.is-offset-one-quarter-touch{margin-left:25%}.column.is-offset-one-fifth-touch{margin-left:20%}.column.is-offset-two-fifths-touch{margin-left:40%}.column.is-offset-three-fifths-touch{margin-left:60%}.column.is-offset-four-fifths-touch{margin-left:80%}.column.is-0-touch{flex:none;width:0%}.column.is-offset-0-touch{margin-left:0}.column.is-1-touch{flex:none;width:8.33333%}.column.is-offset-1-touch{margin-left:8.33333%}.column.is-2-touch{flex:none;width:16.66667%}.column.is-offset-2-touch{margin-left:16.66667%}.column.is-3-touch{flex:none;width:25%}.column.is-offset-3-touch{margin-left:25%}.column.is-4-touch{flex:none;width:33.33333%}.column.is-offset-4-touch{margin-left:33.33333%}.column.is-5-touch{flex:none;width:41.66667%}.column.is-offset-5-touch{margin-left:41.66667%}.column.is-6-touch{flex:none;width:50%}.column.is-offset-6-touch{margin-left:50%}.column.is-7-touch{flex:none;width:58.33333%}.column.is-offset-7-touch{margin-left:58.33333%}.column.is-8-touch{flex:none;width:66.66667%}.column.is-offset-8-touch{margin-left:66.66667%}.column.is-9-touch{flex:none;width:75%}.column.is-offset-9-touch{margin-left:75%}.column.is-10-touch{flex:none;width:83.33333%}.column.is-offset-10-touch{margin-left:83.33333%}.column.is-11-touch{flex:none;width:91.66667%}.column.is-offset-11-touch{margin-left:91.66667%}.column.is-12-touch{flex:none;width:100%}.column.is-offset-12-touch{margin-left:100%}}@media screen and (min-width:1024px){.column.is-narrow-desktop{flex:none}.column.is-full-desktop{flex:none;width:100%}.column.is-three-quarters-desktop{flex:none;width:75%}.column.is-two-thirds-desktop{flex:none;width:66.6666%}.column.is-half-desktop{flex:none;width:50%}.column.is-one-third-desktop{flex:none;width:33.3333%}.column.is-one-quarter-desktop{flex:none;width:25%}.column.is-one-fifth-desktop{flex:none;width:20%}.column.is-two-fifths-desktop{flex:none;width:40%}.column.is-three-fifths-desktop{flex:none;width:60%}.column.is-four-fifths-desktop{flex:none;width:80%}.column.is-offset-three-quarters-desktop{margin-left:75%}.column.is-offset-two-thirds-desktop{margin-left:66.6666%}.column.is-offset-half-desktop{margin-left:50%}.column.is-offset-one-third-desktop{margin-left:33.3333%}.column.is-offset-one-quarter-desktop{margin-left:25%}.column.is-offset-one-fifth-desktop{margin-left:20%}.column.is-offset-two-fifths-desktop{margin-left:40%}.column.is-offset-three-fifths-desktop{margin-left:60%}.column.is-offset-four-fifths-desktop{margin-left:80%}.column.is-0-desktop{flex:none;width:0%}.column.is-offset-0-desktop{margin-left:0}.column.is-1-desktop{flex:none;width:8.33333%}.column.is-offset-1-desktop{margin-left:8.33333%}.column.is-2-desktop{flex:none;width:16.66667%}.column.is-offset-2-desktop{margin-left:16.66667%}.column.is-3-desktop{flex:none;width:25%}.column.is-offset-3-desktop{margin-left:25%}.column.is-4-desktop{flex:none;width:33.33333%}.column.is-offset-4-desktop{margin-left:33.33333%}.column.is-5-desktop{flex:none;width:41.66667%}.column.is-offset-5-desktop{margin-left:41.66667%}.column.is-6-desktop{flex:none;width:50%}.column.is-offset-6-desktop{margin-left:50%}.column.is-7-desktop{flex:none;width:58.33333%}.column.is-offset-7-desktop{margin-left:58.33333%}.column.is-8-desktop{flex:none;width:66.66667%}.column.is-offset-8-desktop{margin-left:66.66667%}.column.is-9-desktop{flex:none;width:75%}.column.is-offset-9-desktop{margin-left:75%}.column.is-10-desktop{flex:none;width:83.33333%}.column.is-offset-10-desktop{margin-left:83.33333%}.column.is-11-desktop{flex:none;width:91.66667%}.column.is-offset-11-desktop{margin-left:91.66667%}.column.is-12-desktop{flex:none;width:100%}.column.is-offset-12-desktop{margin-left:100%}}@media screen and (min-width:1216px){.column.is-narrow-widescreen{flex:none}.column.is-full-widescreen{flex:none;width:100%}.column.is-three-quarters-widescreen{flex:none;width:75%}.column.is-two-thirds-widescreen{flex:none;width:66.6666%}.column.is-half-widescreen{flex:none;width:50%}.column.is-one-third-widescreen{flex:none;width:33.3333%}.column.is-one-quarter-widescreen{flex:none;width:25%}.column.is-one-fifth-widescreen{flex:none;width:20%}.column.is-two-fifths-widescreen{flex:none;width:40%}.column.is-three-fifths-widescreen{flex:none;width:60%}.column.is-four-fifths-widescreen{flex:none;width:80%}.column.is-offset-three-quarters-widescreen{margin-left:75%}.column.is-offset-two-thirds-widescreen{margin-left:66.6666%}.column.is-offset-half-widescreen{margin-left:50%}.column.is-offset-one-third-widescreen{margin-left:33.3333%}.column.is-offset-one-quarter-widescreen{margin-left:25%}.column.is-offset-one-fifth-widescreen{margin-left:20%}.column.is-offset-two-fifths-widescreen{margin-left:40%}.column.is-offset-three-fifths-widescreen{margin-left:60%}.column.is-offset-four-fifths-widescreen{margin-left:80%}.column.is-0-widescreen{flex:none;width:0%}.column.is-offset-0-widescreen{margin-left:0}.column.is-1-widescreen{flex:none;width:8.33333%}.column.is-offset-1-widescreen{margin-left:8.33333%}.column.is-2-widescreen{flex:none;width:16.66667%}.column.is-offset-2-widescreen{margin-left:16.66667%}.column.is-3-widescreen{flex:none;width:25%}.column.is-offset-3-widescreen{margin-left:25%}.column.is-4-widescreen{flex:none;width:33.33333%}.column.is-offset-4-widescreen{margin-left:33.33333%}.column.is-5-widescreen{flex:none;width:41.66667%}.column.is-offset-5-widescreen{margin-left:41.66667%}.column.is-6-widescreen{flex:none;width:50%}.column.is-offset-6-widescreen{margin-left:50%}.column.is-7-widescreen{flex:none;width:58.33333%}.column.is-offset-7-widescreen{margin-left:58.33333%}.column.is-8-widescreen{flex:none;width:66.66667%}.column.is-offset-8-widescreen{margin-left:66.66667%}.column.is-9-widescreen{flex:none;width:75%}.column.is-offset-9-widescreen{margin-left:75%}.column.is-10-widescreen{flex:none;width:83.33333%}.column.is-offset-10-widescreen{margin-left:83.33333%}.column.is-11-widescreen{flex:none;width:91.66667%}.column.is-offset-11-widescreen{margin-left:91.66667%}.column.is-12-widescreen{flex:none;width:100%}.column.is-offset-12-widescreen{margin-left:100%}}@media screen and (min-width:1408px){.column.is-narrow-fullhd{flex:none}.column.is-full-fullhd{flex:none;width:100%}.column.is-three-quarters-fullhd{flex:none;width:75%}.column.is-two-thirds-fullhd{flex:none;width:66.6666%}.column.is-half-fullhd{flex:none;width:50%}.column.is-one-third-fullhd{flex:none;width:33.3333%}.column.is-one-quarter-fullhd{flex:none;width:25%}.column.is-one-fifth-fullhd{flex:none;width:20%}.column.is-two-fifths-fullhd{flex:none;width:40%}.column.is-three-fifths-fullhd{flex:none;width:60%}.column.is-four-fifths-fullhd{flex:none;width:80%}.column.is-offset-three-quarters-fullhd{margin-left:75%}.column.is-offset-two-thirds-fullhd{margin-left:66.6666%}.column.is-offset-half-fullhd{margin-left:50%}.column.is-offset-one-third-fullhd{margin-left:33.3333%}.column.is-offset-one-quarter-fullhd{margin-left:25%}.column.is-offset-one-fifth-fullhd{margin-left:20%}.column.is-offset-two-fifths-fullhd{margin-left:40%}.column.is-offset-three-fifths-fullhd{margin-left:60%}.column.is-offset-four-fifths-fullhd{margin-left:80%}.column.is-0-fullhd{flex:none;width:0%}.column.is-offset-0-fullhd{margin-left:0}.column.is-1-fullhd{flex:none;width:8.33333%}.column.is-offset-1-fullhd{margin-left:8.33333%}.column.is-2-fullhd{flex:none;width:16.66667%}.column.is-offset-2-fullhd{margin-left:16.66667%}.column.is-3-fullhd{flex:none;width:25%}.column.is-offset-3-fullhd{margin-left:25%}.column.is-4-fullhd{flex:none;width:33.33333%}.column.is-offset-4-fullhd{margin-left:33.33333%}.column.is-5-fullhd{flex:none;width:41.66667%}.column.is-offset-5-fullhd{margin-left:41.66667%}.column.is-6-fullhd{flex:none;width:50%}.column.is-offset-6-fullhd{margin-left:50%}.column.is-7-fullhd{flex:none;width:58.33333%}.column.is-offset-7-fullhd{margin-left:58.33333%}.column.is-8-fullhd{flex:none;width:66.66667%}.column.is-offset-8-fullhd{margin-left:66.66667%}.column.is-9-fullhd{flex:none;width:75%}.column.is-offset-9-fullhd{margin-left:75%}.column.is-10-fullhd{flex:none;width:83.33333%}.column.is-offset-10-fullhd{margin-left:83.33333%}.column.is-11-fullhd{flex:none;width:91.66667%}.column.is-offset-11-fullhd{margin-left:91.66667%}.column.is-12-fullhd{flex:none;width:100%}.column.is-offset-12-fullhd{margin-left:100%}}.columns{margin-left:-.75rem;margin-right:-.75rem;margin-top:-.75rem}.columns:last-child{margin-bottom:-.75rem}.columns:not(:last-child){margin-bottom:calc(1.5rem - .75rem)}.columns.is-centered{justify-content:center}.columns.is-gapless{margin-left:0;margin-right:0;margin-top:0}.columns.is-gapless>.column{margin:0;padding:0!important}.columns.is-gapless:not(:last-child){margin-bottom:1.5rem}.columns.is-gapless:last-child{margin-bottom:0}.columns.is-mobile{display:flex}.columns.is-multiline{flex-wrap:wrap}.columns.is-vcentered{align-items:center}@media screen and (min-width:769px),print{.columns:not(.is-desktop){display:flex}}@media screen and (min-width:1024px){.columns.is-desktop{display:flex}}.columns.is-variable{--columnGap:0.75rem;margin-left:calc(-1 * var(--columnGap));margin-right:calc(-1 * var(--columnGap))}.columns.is-variable .column{padding-left:var(--columnGap);padding-right:var(--columnGap)}.columns.is-variable.is-0{--columnGap:0rem}@media screen and (max-width:768px){.columns.is-variable.is-0-mobile{--columnGap:0rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-0-tablet{--columnGap:0rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-0-tablet-only{--columnGap:0rem}}@media screen and (max-width:1023px){.columns.is-variable.is-0-touch{--columnGap:0rem}}@media screen and (min-width:1024px){.columns.is-variable.is-0-desktop{--columnGap:0rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-0-desktop-only{--columnGap:0rem}}@media screen and (min-width:1216px){.columns.is-variable.is-0-widescreen{--columnGap:0rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-0-widescreen-only{--columnGap:0rem}}@media screen and (min-width:1408px){.columns.is-variable.is-0-fullhd{--columnGap:0rem}}.columns.is-variable.is-1{--columnGap:0.25rem}@media screen and (max-width:768px){.columns.is-variable.is-1-mobile{--columnGap:0.25rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-1-tablet{--columnGap:0.25rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-1-tablet-only{--columnGap:0.25rem}}@media screen and (max-width:1023px){.columns.is-variable.is-1-touch{--columnGap:0.25rem}}@media screen and (min-width:1024px){.columns.is-variable.is-1-desktop{--columnGap:0.25rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-1-desktop-only{--columnGap:0.25rem}}@media screen and (min-width:1216px){.columns.is-variable.is-1-widescreen{--columnGap:0.25rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-1-widescreen-only{--columnGap:0.25rem}}@media screen and (min-width:1408px){.columns.is-variable.is-1-fullhd{--columnGap:0.25rem}}.columns.is-variable.is-2{--columnGap:0.5rem}@media screen and (max-width:768px){.columns.is-variable.is-2-mobile{--columnGap:0.5rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-2-tablet{--columnGap:0.5rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-2-tablet-only{--columnGap:0.5rem}}@media screen and (max-width:1023px){.columns.is-variable.is-2-touch{--columnGap:0.5rem}}@media screen and (min-width:1024px){.columns.is-variable.is-2-desktop{--columnGap:0.5rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-2-desktop-only{--columnGap:0.5rem}}@media screen and (min-width:1216px){.columns.is-variable.is-2-widescreen{--columnGap:0.5rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-2-widescreen-only{--columnGap:0.5rem}}@media screen and (min-width:1408px){.columns.is-variable.is-2-fullhd{--columnGap:0.5rem}}.columns.is-variable.is-3{--columnGap:0.75rem}@media screen and (max-width:768px){.columns.is-variable.is-3-mobile{--columnGap:0.75rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-3-tablet{--columnGap:0.75rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-3-tablet-only{--columnGap:0.75rem}}@media screen and (max-width:1023px){.columns.is-variable.is-3-touch{--columnGap:0.75rem}}@media screen and (min-width:1024px){.columns.is-variable.is-3-desktop{--columnGap:0.75rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-3-desktop-only{--columnGap:0.75rem}}@media screen and (min-width:1216px){.columns.is-variable.is-3-widescreen{--columnGap:0.75rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-3-widescreen-only{--columnGap:0.75rem}}@media screen and (min-width:1408px){.columns.is-variable.is-3-fullhd{--columnGap:0.75rem}}.columns.is-variable.is-4{--columnGap:1rem}@media screen and (max-width:768px){.columns.is-variable.is-4-mobile{--columnGap:1rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-4-tablet{--columnGap:1rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-4-tablet-only{--columnGap:1rem}}@media screen and (max-width:1023px){.columns.is-variable.is-4-touch{--columnGap:1rem}}@media screen and (min-width:1024px){.columns.is-variable.is-4-desktop{--columnGap:1rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-4-desktop-only{--columnGap:1rem}}@media screen and (min-width:1216px){.columns.is-variable.is-4-widescreen{--columnGap:1rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-4-widescreen-only{--columnGap:1rem}}@media screen and (min-width:1408px){.columns.is-variable.is-4-fullhd{--columnGap:1rem}}.columns.is-variable.is-5{--columnGap:1.25rem}@media screen and (max-width:768px){.columns.is-variable.is-5-mobile{--columnGap:1.25rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-5-tablet{--columnGap:1.25rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-5-tablet-only{--columnGap:1.25rem}}@media screen and (max-width:1023px){.columns.is-variable.is-5-touch{--columnGap:1.25rem}}@media screen and (min-width:1024px){.columns.is-variable.is-5-desktop{--columnGap:1.25rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-5-desktop-only{--columnGap:1.25rem}}@media screen and (min-width:1216px){.columns.is-variable.is-5-widescreen{--columnGap:1.25rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-5-widescreen-only{--columnGap:1.25rem}}@media screen and (min-width:1408px){.columns.is-variable.is-5-fullhd{--columnGap:1.25rem}}.columns.is-variable.is-6{--columnGap:1.5rem}@media screen and (max-width:768px){.columns.is-variable.is-6-mobile{--columnGap:1.5rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-6-tablet{--columnGap:1.5rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-6-tablet-only{--columnGap:1.5rem}}@media screen and (max-width:1023px){.columns.is-variable.is-6-touch{--columnGap:1.5rem}}@media screen and (min-width:1024px){.columns.is-variable.is-6-desktop{--columnGap:1.5rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-6-desktop-only{--columnGap:1.5rem}}@media screen and (min-width:1216px){.columns.is-variable.is-6-widescreen{--columnGap:1.5rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-6-widescreen-only{--columnGap:1.5rem}}@media screen and (min-width:1408px){.columns.is-variable.is-6-fullhd{--columnGap:1.5rem}}.columns.is-variable.is-7{--columnGap:1.75rem}@media screen and (max-width:768px){.columns.is-variable.is-7-mobile{--columnGap:1.75rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-7-tablet{--columnGap:1.75rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-7-tablet-only{--columnGap:1.75rem}}@media screen and (max-width:1023px){.columns.is-variable.is-7-touch{--columnGap:1.75rem}}@media screen and (min-width:1024px){.columns.is-variable.is-7-desktop{--columnGap:1.75rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-7-desktop-only{--columnGap:1.75rem}}@media screen and (min-width:1216px){.columns.is-variable.is-7-widescreen{--columnGap:1.75rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-7-widescreen-only{--columnGap:1.75rem}}@media screen and (min-width:1408px){.columns.is-variable.is-7-fullhd{--columnGap:1.75rem}}.columns.is-variable.is-8{--columnGap:2rem}@media screen and (max-width:768px){.columns.is-variable.is-8-mobile{--columnGap:2rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-8-tablet{--columnGap:2rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-8-tablet-only{--columnGap:2rem}}@media screen and (max-width:1023px){.columns.is-variable.is-8-touch{--columnGap:2rem}}@media screen and (min-width:1024px){.columns.is-variable.is-8-desktop{--columnGap:2rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-8-desktop-only{--columnGap:2rem}}@media screen and (min-width:1216px){.columns.is-variable.is-8-widescreen{--columnGap:2rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-8-widescreen-only{--columnGap:2rem}}@media screen and (min-width:1408px){.columns.is-variable.is-8-fullhd{--columnGap:2rem}}.tile{align-items:stretch;display:block;flex-basis:0;flex-grow:1;flex-shrink:1;min-height:-webkit-min-content;min-height:-moz-min-content;min-height:min-content}.tile.is-ancestor{margin-left:-.75rem;margin-right:-.75rem;margin-top:-.75rem}.tile.is-ancestor:last-child{margin-bottom:-.75rem}.tile.is-ancestor:not(:last-child){margin-bottom:.75rem}.tile.is-child{margin:0!important}.tile.is-parent{padding:.75rem}.tile.is-vertical{flex-direction:column}.tile.is-vertical>.tile.is-child:not(:last-child){margin-bottom:1.5rem!important}@media screen and (min-width:769px),print{.tile:not(.is-child){display:flex}.tile.is-1{flex:none;width:8.33333%}.tile.is-2{flex:none;width:16.66667%}.tile.is-3{flex:none;width:25%}.tile.is-4{flex:none;width:33.33333%}.tile.is-5{flex:none;width:41.66667%}.tile.is-6{flex:none;width:50%}.tile.is-7{flex:none;width:58.33333%}.tile.is-8{flex:none;width:66.66667%}.tile.is-9{flex:none;width:75%}.tile.is-10{flex:none;width:83.33333%}.tile.is-11{flex:none;width:91.66667%}.tile.is-12{flex:none;width:100%}}.hero{align-items:stretch;display:flex;flex-direction:column;justify-content:space-between}.hero .navbar{background:0 0}.hero .tabs ul{border-bottom:none}.hero.is-white{background-color:#fff;color:#0a0a0a}.hero.is-white a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-white strong{color:inherit}.hero.is-white .title{color:#0a0a0a}.hero.is-white .subtitle{color:rgba(10,10,10,.9)}.hero.is-white .subtitle a:not(.button),.hero.is-white .subtitle strong{color:#0a0a0a}@media screen and (max-width:1023px){.hero.is-white .navbar-menu{background-color:#fff}}.hero.is-white .navbar-item,.hero.is-white .navbar-link{color:rgba(10,10,10,.7)}.hero.is-white .navbar-link.is-active,.hero.is-white .navbar-link:hover,.hero.is-white a.navbar-item.is-active,.hero.is-white a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.hero.is-white .tabs a{color:#0a0a0a;opacity:.9}.hero.is-white .tabs a:hover{opacity:1}.hero.is-white .tabs li.is-active a{opacity:1}.hero.is-white .tabs.is-boxed a,.hero.is-white .tabs.is-toggle a{color:#0a0a0a}.hero.is-white .tabs.is-boxed a:hover,.hero.is-white .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-white .tabs.is-boxed li.is-active a,.hero.is-white .tabs.is-boxed li.is-active a:hover,.hero.is-white .tabs.is-toggle li.is-active a,.hero.is-white .tabs.is-toggle li.is-active a:hover{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.hero.is-white.is-bold{background-image:linear-gradient(141deg,#e6e6e6 0,#fff 71%,#fff 100%)}@media screen and (max-width:768px){.hero.is-white.is-bold .navbar-menu{background-image:linear-gradient(141deg,#e6e6e6 0,#fff 71%,#fff 100%)}}.hero.is-black{background-color:#0a0a0a;color:#fff}.hero.is-black a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-black strong{color:inherit}.hero.is-black .title{color:#fff}.hero.is-black .subtitle{color:rgba(255,255,255,.9)}.hero.is-black .subtitle a:not(.button),.hero.is-black .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-black .navbar-menu{background-color:#0a0a0a}}.hero.is-black .navbar-item,.hero.is-black .navbar-link{color:rgba(255,255,255,.7)}.hero.is-black .navbar-link.is-active,.hero.is-black .navbar-link:hover,.hero.is-black a.navbar-item.is-active,.hero.is-black a.navbar-item:hover{background-color:#000;color:#fff}.hero.is-black .tabs a{color:#fff;opacity:.9}.hero.is-black .tabs a:hover{opacity:1}.hero.is-black .tabs li.is-active a{opacity:1}.hero.is-black .tabs.is-boxed a,.hero.is-black .tabs.is-toggle a{color:#fff}.hero.is-black .tabs.is-boxed a:hover,.hero.is-black .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-black .tabs.is-boxed li.is-active a,.hero.is-black .tabs.is-boxed li.is-active a:hover,.hero.is-black .tabs.is-toggle li.is-active a,.hero.is-black .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#0a0a0a}.hero.is-black.is-bold{background-image:linear-gradient(141deg,#000 0,#0a0a0a 71%,#181616 100%)}@media screen and (max-width:768px){.hero.is-black.is-bold .navbar-menu{background-image:linear-gradient(141deg,#000 0,#0a0a0a 71%,#181616 100%)}}.hero.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.hero.is-light a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-light strong{color:inherit}.hero.is-light .title{color:rgba(0,0,0,.7)}.hero.is-light .subtitle{color:rgba(0,0,0,.9)}.hero.is-light .subtitle a:not(.button),.hero.is-light .subtitle strong{color:rgba(0,0,0,.7)}@media screen and (max-width:1023px){.hero.is-light .navbar-menu{background-color:#f5f5f5}}.hero.is-light .navbar-item,.hero.is-light .navbar-link{color:rgba(0,0,0,.7)}.hero.is-light .navbar-link.is-active,.hero.is-light .navbar-link:hover,.hero.is-light a.navbar-item.is-active,.hero.is-light a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.hero.is-light .tabs a{color:rgba(0,0,0,.7);opacity:.9}.hero.is-light .tabs a:hover{opacity:1}.hero.is-light .tabs li.is-active a{opacity:1}.hero.is-light .tabs.is-boxed a,.hero.is-light .tabs.is-toggle a{color:rgba(0,0,0,.7)}.hero.is-light .tabs.is-boxed a:hover,.hero.is-light .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-light .tabs.is-boxed li.is-active a,.hero.is-light .tabs.is-boxed li.is-active a:hover,.hero.is-light .tabs.is-toggle li.is-active a,.hero.is-light .tabs.is-toggle li.is-active a:hover{background-color:rgba(0,0,0,.7);border-color:rgba(0,0,0,.7);color:#f5f5f5}.hero.is-light.is-bold{background-image:linear-gradient(141deg,#dfd8d9 0,#f5f5f5 71%,#fff 100%)}@media screen and (max-width:768px){.hero.is-light.is-bold .navbar-menu{background-image:linear-gradient(141deg,#dfd8d9 0,#f5f5f5 71%,#fff 100%)}}.hero.is-dark{background-color:#363636;color:#fff}.hero.is-dark a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-dark strong{color:inherit}.hero.is-dark .title{color:#fff}.hero.is-dark .subtitle{color:rgba(255,255,255,.9)}.hero.is-dark .subtitle a:not(.button),.hero.is-dark .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-dark .navbar-menu{background-color:#363636}}.hero.is-dark .navbar-item,.hero.is-dark .navbar-link{color:rgba(255,255,255,.7)}.hero.is-dark .navbar-link.is-active,.hero.is-dark .navbar-link:hover,.hero.is-dark a.navbar-item.is-active,.hero.is-dark a.navbar-item:hover{background-color:#292929;color:#fff}.hero.is-dark .tabs a{color:#fff;opacity:.9}.hero.is-dark .tabs a:hover{opacity:1}.hero.is-dark .tabs li.is-active a{opacity:1}.hero.is-dark .tabs.is-boxed a,.hero.is-dark .tabs.is-toggle a{color:#fff}.hero.is-dark .tabs.is-boxed a:hover,.hero.is-dark .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-dark .tabs.is-boxed li.is-active a,.hero.is-dark .tabs.is-boxed li.is-active a:hover,.hero.is-dark .tabs.is-toggle li.is-active a,.hero.is-dark .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#363636}.hero.is-dark.is-bold{background-image:linear-gradient(141deg,#1f191a 0,#363636 71%,#46403f 100%)}@media screen and (max-width:768px){.hero.is-dark.is-bold .navbar-menu{background-image:linear-gradient(141deg,#1f191a 0,#363636 71%,#46403f 100%)}}.hero.is-primary{background-color:#00d1b2;color:#fff}.hero.is-primary a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-primary strong{color:inherit}.hero.is-primary .title{color:#fff}.hero.is-primary .subtitle{color:rgba(255,255,255,.9)}.hero.is-primary .subtitle a:not(.button),.hero.is-primary .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-primary .navbar-menu{background-color:#00d1b2}}.hero.is-primary .navbar-item,.hero.is-primary .navbar-link{color:rgba(255,255,255,.7)}.hero.is-primary .navbar-link.is-active,.hero.is-primary .navbar-link:hover,.hero.is-primary a.navbar-item.is-active,.hero.is-primary a.navbar-item:hover{background-color:#00b89c;color:#fff}.hero.is-primary .tabs a{color:#fff;opacity:.9}.hero.is-primary .tabs a:hover{opacity:1}.hero.is-primary .tabs li.is-active a{opacity:1}.hero.is-primary .tabs.is-boxed a,.hero.is-primary .tabs.is-toggle a{color:#fff}.hero.is-primary .tabs.is-boxed a:hover,.hero.is-primary .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-primary .tabs.is-boxed li.is-active a,.hero.is-primary .tabs.is-boxed li.is-active a:hover,.hero.is-primary .tabs.is-toggle li.is-active a,.hero.is-primary .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#00d1b2}.hero.is-primary.is-bold{background-image:linear-gradient(141deg,#009e6c 0,#00d1b2 71%,#00e7eb 100%)}@media screen and (max-width:768px){.hero.is-primary.is-bold .navbar-menu{background-image:linear-gradient(141deg,#009e6c 0,#00d1b2 71%,#00e7eb 100%)}}.hero.is-link{background-color:#3273dc;color:#fff}.hero.is-link a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-link strong{color:inherit}.hero.is-link .title{color:#fff}.hero.is-link .subtitle{color:rgba(255,255,255,.9)}.hero.is-link .subtitle a:not(.button),.hero.is-link .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-link .navbar-menu{background-color:#3273dc}}.hero.is-link .navbar-item,.hero.is-link .navbar-link{color:rgba(255,255,255,.7)}.hero.is-link .navbar-link.is-active,.hero.is-link .navbar-link:hover,.hero.is-link a.navbar-item.is-active,.hero.is-link a.navbar-item:hover{background-color:#2366d1;color:#fff}.hero.is-link .tabs a{color:#fff;opacity:.9}.hero.is-link .tabs a:hover{opacity:1}.hero.is-link .tabs li.is-active a{opacity:1}.hero.is-link .tabs.is-boxed a,.hero.is-link .tabs.is-toggle a{color:#fff}.hero.is-link .tabs.is-boxed a:hover,.hero.is-link .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-link .tabs.is-boxed li.is-active a,.hero.is-link .tabs.is-boxed li.is-active a:hover,.hero.is-link .tabs.is-toggle li.is-active a,.hero.is-link .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#3273dc}.hero.is-link.is-bold{background-image:linear-gradient(141deg,#1577c6 0,#3273dc 71%,#4366e5 100%)}@media screen and (max-width:768px){.hero.is-link.is-bold .navbar-menu{background-image:linear-gradient(141deg,#1577c6 0,#3273dc 71%,#4366e5 100%)}}.hero.is-info{background-color:#3298dc;color:#fff}.hero.is-info a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-info strong{color:inherit}.hero.is-info .title{color:#fff}.hero.is-info .subtitle{color:rgba(255,255,255,.9)}.hero.is-info .subtitle a:not(.button),.hero.is-info .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-info .navbar-menu{background-color:#3298dc}}.hero.is-info .navbar-item,.hero.is-info .navbar-link{color:rgba(255,255,255,.7)}.hero.is-info .navbar-link.is-active,.hero.is-info .navbar-link:hover,.hero.is-info a.navbar-item.is-active,.hero.is-info a.navbar-item:hover{background-color:#238cd1;color:#fff}.hero.is-info .tabs a{color:#fff;opacity:.9}.hero.is-info .tabs a:hover{opacity:1}.hero.is-info .tabs li.is-active a{opacity:1}.hero.is-info .tabs.is-boxed a,.hero.is-info .tabs.is-toggle a{color:#fff}.hero.is-info .tabs.is-boxed a:hover,.hero.is-info .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-info .tabs.is-boxed li.is-active a,.hero.is-info .tabs.is-boxed li.is-active a:hover,.hero.is-info .tabs.is-toggle li.is-active a,.hero.is-info .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#3298dc}.hero.is-info.is-bold{background-image:linear-gradient(141deg,#159dc6 0,#3298dc 71%,#4389e5 100%)}@media screen and (max-width:768px){.hero.is-info.is-bold .navbar-menu{background-image:linear-gradient(141deg,#159dc6 0,#3298dc 71%,#4389e5 100%)}}.hero.is-success{background-color:#48c774;color:#fff}.hero.is-success a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-success strong{color:inherit}.hero.is-success .title{color:#fff}.hero.is-success .subtitle{color:rgba(255,255,255,.9)}.hero.is-success .subtitle a:not(.button),.hero.is-success .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-success .navbar-menu{background-color:#48c774}}.hero.is-success .navbar-item,.hero.is-success .navbar-link{color:rgba(255,255,255,.7)}.hero.is-success .navbar-link.is-active,.hero.is-success .navbar-link:hover,.hero.is-success a.navbar-item.is-active,.hero.is-success a.navbar-item:hover{background-color:#3abb67;color:#fff}.hero.is-success .tabs a{color:#fff;opacity:.9}.hero.is-success .tabs a:hover{opacity:1}.hero.is-success .tabs li.is-active a{opacity:1}.hero.is-success .tabs.is-boxed a,.hero.is-success .tabs.is-toggle a{color:#fff}.hero.is-success .tabs.is-boxed a:hover,.hero.is-success .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-success .tabs.is-boxed li.is-active a,.hero.is-success .tabs.is-boxed li.is-active a:hover,.hero.is-success .tabs.is-toggle li.is-active a,.hero.is-success .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#48c774}.hero.is-success.is-bold{background-image:linear-gradient(141deg,#29b342 0,#48c774 71%,#56d296 100%)}@media screen and (max-width:768px){.hero.is-success.is-bold .navbar-menu{background-image:linear-gradient(141deg,#29b342 0,#48c774 71%,#56d296 100%)}}.hero.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.hero.is-warning a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-warning strong{color:inherit}.hero.is-warning .title{color:rgba(0,0,0,.7)}.hero.is-warning .subtitle{color:rgba(0,0,0,.9)}.hero.is-warning .subtitle a:not(.button),.hero.is-warning .subtitle strong{color:rgba(0,0,0,.7)}@media screen and (max-width:1023px){.hero.is-warning .navbar-menu{background-color:#ffdd57}}.hero.is-warning .navbar-item,.hero.is-warning .navbar-link{color:rgba(0,0,0,.7)}.hero.is-warning .navbar-link.is-active,.hero.is-warning .navbar-link:hover,.hero.is-warning a.navbar-item.is-active,.hero.is-warning a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.hero.is-warning .tabs a{color:rgba(0,0,0,.7);opacity:.9}.hero.is-warning .tabs a:hover{opacity:1}.hero.is-warning .tabs li.is-active a{opacity:1}.hero.is-warning .tabs.is-boxed a,.hero.is-warning .tabs.is-toggle a{color:rgba(0,0,0,.7)}.hero.is-warning .tabs.is-boxed a:hover,.hero.is-warning .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-warning .tabs.is-boxed li.is-active a,.hero.is-warning .tabs.is-boxed li.is-active a:hover,.hero.is-warning .tabs.is-toggle li.is-active a,.hero.is-warning .tabs.is-toggle li.is-active a:hover{background-color:rgba(0,0,0,.7);border-color:rgba(0,0,0,.7);color:#ffdd57}.hero.is-warning.is-bold{background-image:linear-gradient(141deg,#ffaf24 0,#ffdd57 71%,#fffa70 100%)}@media screen and (max-width:768px){.hero.is-warning.is-bold .navbar-menu{background-image:linear-gradient(141deg,#ffaf24 0,#ffdd57 71%,#fffa70 100%)}}.hero.is-danger{background-color:#f14668;color:#fff}.hero.is-danger a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-danger strong{color:inherit}.hero.is-danger .title{color:#fff}.hero.is-danger .subtitle{color:rgba(255,255,255,.9)}.hero.is-danger .subtitle a:not(.button),.hero.is-danger .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-danger .navbar-menu{background-color:#f14668}}.hero.is-danger .navbar-item,.hero.is-danger .navbar-link{color:rgba(255,255,255,.7)}.hero.is-danger .navbar-link.is-active,.hero.is-danger .navbar-link:hover,.hero.is-danger a.navbar-item.is-active,.hero.is-danger a.navbar-item:hover{background-color:#ef2e55;color:#fff}.hero.is-danger .tabs a{color:#fff;opacity:.9}.hero.is-danger .tabs a:hover{opacity:1}.hero.is-danger .tabs li.is-active a{opacity:1}.hero.is-danger .tabs.is-boxed a,.hero.is-danger .tabs.is-toggle a{color:#fff}.hero.is-danger .tabs.is-boxed a:hover,.hero.is-danger .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-danger .tabs.is-boxed li.is-active a,.hero.is-danger .tabs.is-boxed li.is-active a:hover,.hero.is-danger .tabs.is-toggle li.is-active a,.hero.is-danger .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#f14668}.hero.is-danger.is-bold{background-image:linear-gradient(141deg,#fa0a62 0,#f14668 71%,#f7595f 100%)}@media screen and (max-width:768px){.hero.is-danger.is-bold .navbar-menu{background-image:linear-gradient(141deg,#fa0a62 0,#f14668 71%,#f7595f 100%)}}.hero.is-small .hero-body{padding-bottom:1.5rem;padding-top:1.5rem}@media screen and (min-width:769px),print{.hero.is-medium .hero-body{padding-bottom:9rem;padding-top:9rem}}@media screen and (min-width:769px),print{.hero.is-large .hero-body{padding-bottom:18rem;padding-top:18rem}}.hero.is-fullheight .hero-body,.hero.is-fullheight-with-navbar .hero-body,.hero.is-halfheight .hero-body{align-items:center;display:flex}.hero.is-fullheight .hero-body>.container,.hero.is-fullheight-with-navbar .hero-body>.container,.hero.is-halfheight .hero-body>.container{flex-grow:1;flex-shrink:1}.hero.is-halfheight{min-height:50vh}.hero.is-fullheight{min-height:100vh}.hero-video{overflow:hidden}.hero-video video{left:50%;min-height:100%;min-width:100%;position:absolute;top:50%;transform:translate3d(-50%,-50%,0)}.hero-video.is-transparent{opacity:.3}@media screen and (max-width:768px){.hero-video{display:none}}.hero-buttons{margin-top:1.5rem}@media screen and (max-width:768px){.hero-buttons .button{display:flex}.hero-buttons .button:not(:last-child){margin-bottom:.75rem}}@media screen and (min-width:769px),print{.hero-buttons{display:flex;justify-content:center}.hero-buttons .button:not(:last-child){margin-right:1.5rem}}.hero-foot,.hero-head{flex-grow:0;flex-shrink:0}.hero-body{flex-grow:1;flex-shrink:0;padding:3rem 1.5rem}.section{padding:3rem 1.5rem}@media screen and (min-width:1024px){.section.is-medium{padding:9rem 1.5rem}.section.is-large{padding:18rem 1.5rem}}.footer{background-color:#fafafa;padding:3rem 1.5rem 6rem} \ No newline at end of file diff --git a/public/interface.html b/public/interface.html new file mode 100644 index 000000000..ca3005392 --- /dev/null +++ b/public/interface.html @@ -0,0 +1,333 @@ + + + + + + + MeiliSearch + + + + +
+ +
+
+
+

+ Welcome to MeiliSearch +

+

+ This dashboard will help you check the search results with ease. +

+
+
+
+
+ + +
+ +
+

At least a private API key is required for the dashboard to access the indexes list.

+
+
+
+
+
+ +
+
+ + + +
+
+ +
+
+
+
+
+
+

Documents

+

0

+
+
+

Time Spent

+

N/A

+
+
+
+
+
+
+
+ +
+
+
    + +
+
+
+ + + + diff --git a/src/analytics.rs b/src/analytics.rs new file mode 100644 index 000000000..379a25030 --- /dev/null +++ b/src/analytics.rs @@ -0,0 +1,137 @@ +use std::hash::{Hash, Hasher}; +use std::{error, thread}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use log::error; +use serde::Serialize; +use serde_qs as qs; +use siphasher::sip::SipHasher; +use walkdir::WalkDir; + +use crate::Data; +use crate::Opt; + +const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47"; + +#[derive(Debug, Serialize)] +struct EventProperties { + database_size: u64, + last_update_timestamp: Option, //timestamp + number_of_documents: Vec, +} + +impl EventProperties { + fn from(data: Data) -> Result> { + let mut index_list = Vec::new(); + + let reader = data.db.main_read_txn()?; + + for index_uid in data.db.indexes_uids() { + if let Some(index) = data.db.open_index(&index_uid) { + let number_of_documents = index.main.number_of_documents(&reader)?; + index_list.push(number_of_documents); + } + } + + let database_size = WalkDir::new(&data.db_path) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len()); + + let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp()); + + Ok(EventProperties { + database_size, + last_update_timestamp, + number_of_documents: index_list, + }) + } +} + +#[derive(Debug, Serialize)] +struct UserProperties<'a> { + env: &'a str, + start_since_days: u64, + user_email: Option, + server_provider: Option, +} + +#[derive(Debug, Serialize)] +struct Event<'a> { + user_id: &'a str, + event_type: &'a str, + device_id: &'a str, + time: u64, + app_version: &'a str, + user_properties: UserProperties<'a>, + event_properties: Option, +} + +#[derive(Debug, Serialize)] +struct AmplitudeRequest<'a> { + api_key: &'a str, + event: &'a str, +} + +pub fn analytics_sender(data: Data, opt: Opt) { + let username = whoami::username(); + let hostname = whoami::hostname(); + let platform = whoami::platform(); + + let uid = username + &hostname + &platform.to_string(); + + let mut hasher = SipHasher::new(); + uid.hash(&mut hasher); + let hash = hasher.finish(); + + let uid = format!("{:X}", hash); + let platform = platform.to_string(); + let first_start = Instant::now(); + + loop { + let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + let user_id = &uid; + let device_id = &platform; + let time = n.as_secs(); + let event_type = "runtime_tick"; + let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day + let event_properties = EventProperties::from(data.clone()).ok(); + let app_version = env!("CARGO_PKG_VERSION").to_string(); + let app_version = app_version.as_str(); + let user_email = std::env::var("MEILI_USER_EMAIL").ok(); + let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok(); + let user_properties = UserProperties { + env: &opt.env, + start_since_days: elapsed_since_start, + user_email, + server_provider, + }; + + let event = Event { + user_id, + event_type, + device_id, + time, + app_version, + user_properties, + event_properties + }; + let event = serde_json::to_string(&event).unwrap(); + + let request = AmplitudeRequest { + api_key: AMPLITUDE_API_KEY, + event: &event, + }; + + let body = qs::to_string(&request).unwrap(); + let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body); + if !response.ok() { + let body = response.into_string().unwrap(); + error!("Unsuccessful call to Amplitude: {}", body); + } + + thread::sleep(Duration::from_secs(3600)) // one hour + } +} diff --git a/src/data.rs b/src/data.rs new file mode 100644 index 000000000..783c81fd8 --- /dev/null +++ b/src/data.rs @@ -0,0 +1,162 @@ +use std::error::Error; +use std::ops::Deref; +use std::path::PathBuf; +use std::sync::Arc; + +use meilisearch_core::{Database, DatabaseOptions, Index}; +use sha2::Digest; + +use crate::error::{Error as MSError, ResponseError}; +use crate::index_update_callback; +use crate::option::Opt; + +#[derive(Clone)] +pub struct Data { + inner: Arc, +} + +impl Deref for Data { + type Target = DataInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[derive(Clone)] +pub struct DataInner { + pub db: Arc, + pub db_path: String, + pub dumps_dir: PathBuf, + pub dump_batch_size: usize, + pub api_keys: ApiKeys, + pub server_pid: u32, + pub http_payload_size_limit: usize, +} + +#[derive(Clone)] +pub struct ApiKeys { + pub public: Option, + pub private: Option, + pub master: Option, +} + +impl ApiKeys { + pub fn generate_missing_api_keys(&mut self) { + if let Some(master_key) = &self.master { + if self.private.is_none() { + let key = format!("{}-private", master_key); + let sha = sha2::Sha256::digest(key.as_bytes()); + self.private = Some(format!("{:x}", sha)); + } + if self.public.is_none() { + let key = format!("{}-public", master_key); + let sha = sha2::Sha256::digest(key.as_bytes()); + self.public = Some(format!("{:x}", sha)); + } + } + } +} + +impl Data { + pub fn new(opt: Opt) -> Result> { + let db_path = opt.db_path.clone(); + let dumps_dir = opt.dumps_dir.clone(); + let dump_batch_size = opt.dump_batch_size; + let server_pid = std::process::id(); + + let db_opt = DatabaseOptions { + main_map_size: opt.max_mdb_size, + update_map_size: opt.max_udb_size, + }; + + let http_payload_size_limit = opt.http_payload_size_limit; + + let db = Arc::new(Database::open_or_create(opt.db_path, db_opt)?); + + let mut api_keys = ApiKeys { + master: opt.master_key, + private: None, + public: None, + }; + + api_keys.generate_missing_api_keys(); + + let inner_data = DataInner { + db: db.clone(), + db_path, + dumps_dir, + dump_batch_size, + api_keys, + server_pid, + http_payload_size_limit, + }; + + let data = Data { + inner: Arc::new(inner_data), + }; + + let callback_context = data.clone(); + db.set_update_callback(Box::new(move |index_uid, status| { + index_update_callback(&index_uid, &callback_context, status); + })); + + Ok(data) + } + + fn create_index(&self, uid: &str) -> Result { + if !uid + .chars() + .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') + { + return Err(MSError::InvalidIndexUid.into()); + } + + let created_index = self.db.create_index(&uid).map_err(|e| match e { + meilisearch_core::Error::IndexAlreadyExists => e.into(), + _ => ResponseError::from(MSError::create_index(e)), + })?; + + self.db.main_write::<_, _, ResponseError>(|mut writer| { + created_index.main.put_name(&mut writer, uid)?; + + created_index + .main + .created_at(&writer)? + .ok_or(MSError::internal("Impossible to read created at"))?; + + created_index + .main + .updated_at(&writer)? + .ok_or(MSError::internal("Impossible to read updated at"))?; + Ok(()) + })?; + + Ok(created_index) + } + + pub fn get_or_create_index(&self, uid: &str, f: F) -> Result + where + F: FnOnce(&Index) -> Result, + { + let mut index_has_been_created = false; + + let index = match self.db.open_index(&uid) { + Some(index) => index, + None => { + index_has_been_created = true; + self.create_index(&uid)? + } + }; + + match f(&index) { + Ok(r) => Ok(r), + Err(err) => { + if index_has_been_created { + let _ = self.db.delete_index(&uid); + } + Err(err) + } + } + } +} diff --git a/src/dump.rs b/src/dump.rs new file mode 100644 index 000000000..468dbf640 --- /dev/null +++ b/src/dump.rs @@ -0,0 +1,425 @@ +use std::fs::{create_dir_all, File}; +use std::io::prelude::*; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::thread; + +use actix_web::web; +use chrono::offset::Utc; +use indexmap::IndexMap; +use log::{error, info}; +use meilisearch_core::{MainWriter, MainReader, UpdateReader}; +use meilisearch_core::settings::Settings; +use meilisearch_core::update::{apply_settings_update, apply_documents_addition}; +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use tempfile::TempDir; + +use crate::Data; +use crate::error::{Error, ResponseError}; +use crate::helpers::compression; +use crate::routes::index; +use crate::routes::index::IndexResponse; + +// Mutex to share dump progress. +static DUMP_INFO: Lazy>> = Lazy::new(Mutex::default); + +#[derive(Debug, Serialize, Deserialize, Copy, Clone)] +enum DumpVersion { + V1, +} + +impl DumpVersion { + const CURRENT: Self = Self::V1; +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DumpMetadata { + indexes: Vec, + db_version: String, + dump_version: DumpVersion, +} + +impl DumpMetadata { + /// Create a DumpMetadata with the current dump version of meilisearch. + pub fn new(indexes: Vec, db_version: String) -> Self { + DumpMetadata { + indexes, + db_version, + dump_version: DumpVersion::CURRENT, + } + } + + /// Extract DumpMetadata from `metadata.json` file present at provided `dir_path` + fn from_path(dir_path: &Path) -> Result { + let path = dir_path.join("metadata.json"); + let file = File::open(path)?; + let reader = std::io::BufReader::new(file); + let metadata = serde_json::from_reader(reader)?; + + Ok(metadata) + } + + /// Write DumpMetadata in `metadata.json` file at provided `dir_path` + fn to_path(&self, dir_path: &Path) -> Result<(), Error> { + let path = dir_path.join("metadata.json"); + let file = File::create(path)?; + + serde_json::to_writer(file, &self)?; + + Ok(()) + } +} + +/// Extract Settings from `settings.json` file present at provided `dir_path` +fn settings_from_path(dir_path: &Path) -> Result { + let path = dir_path.join("settings.json"); + let file = File::open(path)?; + let reader = std::io::BufReader::new(file); + let metadata = serde_json::from_reader(reader)?; + + Ok(metadata) +} + +/// Write Settings in `settings.json` file at provided `dir_path` +fn settings_to_path(settings: &Settings, dir_path: &Path) -> Result<(), Error> { + let path = dir_path.join("settings.json"); + let file = File::create(path)?; + + serde_json::to_writer(file, settings)?; + + Ok(()) +} + +/// Import settings and documents of a dump with version `DumpVersion::V1` in specified index. +fn import_index_v1( + data: &Data, + dumps_dir: &Path, + index_uid: &str, + document_batch_size: usize, + write_txn: &mut MainWriter, +) -> Result<(), Error> { + + // open index + let index = data + .db + .open_index(index_uid) + .ok_or(Error::index_not_found(index_uid))?; + + // index dir path in dump dir + let index_path = &dumps_dir.join(index_uid); + + // extract `settings.json` file and import content + let settings = settings_from_path(&index_path)?; + let settings = settings.to_update().map_err(|e| Error::dump_failed(format!("importing settings for index {}; {}", index_uid, e)))?; + apply_settings_update(write_txn, &index, settings)?; + + // create iterator over documents in `documents.jsonl` to make batch importation + // create iterator over documents in `documents.jsonl` to make batch importation + let documents = { + let file = File::open(&index_path.join("documents.jsonl"))?; + let reader = std::io::BufReader::new(file); + let deserializer = serde_json::Deserializer::from_reader(reader); + deserializer.into_iter::>() + }; + + // batch import document every `document_batch_size`: + // create a Vec to bufferize documents + let mut values = Vec::with_capacity(document_batch_size); + // iterate over documents + for document in documents { + // push document in buffer + values.push(document?); + // if buffer is full, create and apply a batch, and clean buffer + if values.len() == document_batch_size { + let batch = std::mem::replace(&mut values, Vec::with_capacity(document_batch_size)); + apply_documents_addition(write_txn, &index, batch)?; + } + } + + // apply documents remaining in the buffer + if !values.is_empty() { + apply_documents_addition(write_txn, &index, values)?; + } + + // sync index information: stats, updated_at, last_update + if let Err(e) = crate::index_update_callback_txn(index, index_uid, data, write_txn) { + return Err(Error::Internal(e)); + } + + Ok(()) +} + +/// Import dump from `dump_path` in database. +pub fn import_dump( + data: &Data, + dump_path: &Path, + document_batch_size: usize, +) -> Result<(), Error> { + info!("Importing dump from {:?}...", dump_path); + + // create a temporary directory + let tmp_dir = TempDir::new()?; + let tmp_dir_path = tmp_dir.path(); + + // extract dump in temporary directory + compression::from_tar_gz(dump_path, tmp_dir_path)?; + + // read dump metadata + let metadata = DumpMetadata::from_path(&tmp_dir_path)?; + + // choose importation function from DumpVersion of metadata + let import_index = match metadata.dump_version { + DumpVersion::V1 => import_index_v1, + }; + + // remove indexes which have same `uid` than indexes to import and create empty indexes + let existing_index_uids = data.db.indexes_uids(); + for index in metadata.indexes.iter() { + if existing_index_uids.contains(&index.uid) { + data.db.delete_index(index.uid.clone())?; + } + index::create_index_sync(&data.db, index.uid.clone(), index.name.clone(), index.primary_key.clone())?; + } + + // import each indexes content + data.db.main_write::<_, _, Error>(|mut writer| { + for index in metadata.indexes { + import_index(&data, tmp_dir_path, &index.uid, document_batch_size, &mut writer)?; + } + Ok(()) + })?; + + info!("Dump importation from {:?} succeed", dump_path); + Ok(()) +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +#[serde(rename_all = "snake_case")] +pub enum DumpStatus { + Done, + InProgress, + Failed, +} + +#[derive(Debug, Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct DumpInfo { + pub uid: String, + pub status: DumpStatus, + #[serde(skip_serializing_if = "Option::is_none", flatten)] + pub error: Option, +} + +impl DumpInfo { + pub fn new(uid: String, status: DumpStatus) -> Self { + Self { uid, status, error: None } + } + + pub fn with_error(mut self, error: ResponseError) -> Self { + self.status = DumpStatus::Failed; + self.error = Some(json!(error)); + + self + } + + pub fn dump_already_in_progress(&self) -> bool { + self.status == DumpStatus::InProgress + } + + pub fn get_current() -> Option { + DUMP_INFO.lock().unwrap().clone() + } + + pub fn set_current(&self) { + *DUMP_INFO.lock().unwrap() = Some(self.clone()); + } +} + +/// Generate uid from creation date +fn generate_uid() -> String { + Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() +} + +/// Infer dumps_dir from dump_uid +pub fn compressed_dumps_dir(dumps_dir: &Path, dump_uid: &str) -> PathBuf { + dumps_dir.join(format!("{}.dump", dump_uid)) +} + +/// Write metadata in dump +fn dump_metadata(data: &web::Data, dir_path: &Path, indexes: Vec) -> Result<(), Error> { + let (db_major, db_minor, db_patch) = data.db.version(); + let metadata = DumpMetadata::new(indexes, format!("{}.{}.{}", db_major, db_minor, db_patch)); + + metadata.to_path(dir_path) +} + +/// Export settings of provided index in dump +fn dump_index_settings(data: &web::Data, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { + let settings = crate::routes::setting::get_all_sync(data, reader, index_uid)?; + + settings_to_path(&settings, dir_path) +} + +/// Export updates of provided index in dump +fn dump_index_updates(data: &web::Data, reader: &UpdateReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { + let updates_path = dir_path.join("updates.jsonl"); + let updates = crate::routes::index::get_all_updates_status_sync(data, reader, index_uid)?; + + let file = File::create(updates_path)?; + + for update in updates { + serde_json::to_writer(&file, &update)?; + writeln!(&file)?; + } + + Ok(()) +} + +/// Export documents of provided index in dump +fn dump_index_documents(data: &web::Data, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { + let documents_path = dir_path.join("documents.jsonl"); + let file = File::create(documents_path)?; + let dump_batch_size = data.dump_batch_size; + + let mut offset = 0; + loop { + let documents = crate::routes::document::get_all_documents_sync(data, reader, index_uid, offset, dump_batch_size, None)?; + if documents.is_empty() { break; } else { offset += dump_batch_size; } + + for document in documents { + serde_json::to_writer(&file, &document)?; + writeln!(&file)?; + } + } + + Ok(()) +} + +/// Write error with a context. +fn fail_dump_process(dump_info: DumpInfo, context: &str, error: E) { + let error_message = format!("{}; {}", context, error); + + error!("Something went wrong during dump process: {}", &error_message); + dump_info.with_error(Error::dump_failed(error_message).into()).set_current(); +} + +/// Main function of dump. +fn dump_process(data: web::Data, dumps_dir: PathBuf, dump_info: DumpInfo) { + // open read transaction on Update + let update_reader = match data.db.update_read_txn() { + Ok(r) => r, + Err(e) => { + fail_dump_process(dump_info, "creating RO transaction on updates", e); + return ; + } + }; + + // open read transaction on Main + let main_reader = match data.db.main_read_txn() { + Ok(r) => r, + Err(e) => { + fail_dump_process(dump_info, "creating RO transaction on main", e); + return ; + } + }; + + // create a temporary directory + let tmp_dir = match TempDir::new() { + Ok(tmp_dir) => tmp_dir, + Err(e) => { + fail_dump_process(dump_info, "creating temporary directory", e); + return ; + } + }; + let tmp_dir_path = tmp_dir.path(); + + // fetch indexes + let indexes = match crate::routes::index::list_indexes_sync(&data, &main_reader) { + Ok(indexes) => indexes, + Err(e) => { + fail_dump_process(dump_info, "listing indexes", e); + return ; + } + }; + + // create metadata + if let Err(e) = dump_metadata(&data, &tmp_dir_path, indexes.clone()) { + fail_dump_process(dump_info, "generating metadata", e); + return ; + } + + // export settings, updates and documents for each indexes + for index in indexes { + let index_path = tmp_dir_path.join(&index.uid); + + // create index sub-dircetory + if let Err(e) = create_dir_all(&index_path) { + fail_dump_process(dump_info, &format!("creating directory for index {}", &index.uid), e); + return ; + } + + // export settings + if let Err(e) = dump_index_settings(&data, &main_reader, &index_path, &index.uid) { + fail_dump_process(dump_info, &format!("generating settings for index {}", &index.uid), e); + return ; + } + + // export documents + if let Err(e) = dump_index_documents(&data, &main_reader, &index_path, &index.uid) { + fail_dump_process(dump_info, &format!("generating documents for index {}", &index.uid), e); + return ; + } + + // export updates + if let Err(e) = dump_index_updates(&data, &update_reader, &index_path, &index.uid) { + fail_dump_process(dump_info, &format!("generating updates for index {}", &index.uid), e); + return ; + } + } + + // compress dump in a file named `{dump_uid}.dump` in `dumps_dir` + if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_dir(&dumps_dir, &dump_info.uid)) { + fail_dump_process(dump_info, "compressing dump", e); + return ; + } + + // update dump info to `done` + let resume = DumpInfo::new( + dump_info.uid, + DumpStatus::Done + ); + + resume.set_current(); +} + +pub fn init_dump_process(data: &web::Data, dumps_dir: &Path) -> Result { + create_dir_all(dumps_dir).map_err(|e| Error::dump_failed(format!("creating temporary directory {}", e)))?; + + // check if a dump is already in progress + if let Some(resume) = DumpInfo::get_current() { + if resume.dump_already_in_progress() { + return Err(Error::dump_conflict()) + } + } + + // generate a new dump info + let info = DumpInfo::new( + generate_uid(), + DumpStatus::InProgress + ); + + info.set_current(); + + let data = data.clone(); + let dumps_dir = dumps_dir.to_path_buf(); + let info_cloned = info.clone(); + // run dump process in a new thread + thread::spawn(move || + dump_process(data, dumps_dir, info_cloned) + ); + + Ok(info) +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 000000000..e779c5708 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,307 @@ +use std::error; +use std::fmt; + +use actix_http::ResponseBuilder; +use actix_web as aweb; +use actix_web::error::{JsonPayloadError, QueryPayloadError}; +use actix_web::http::StatusCode; +use serde::ser::{Serialize, Serializer, SerializeStruct}; + +use meilisearch_error::{ErrorCode, Code}; + +#[derive(Debug)] +pub struct ResponseError { + inner: Box, +} + +impl error::Error for ResponseError {} + +impl ErrorCode for ResponseError { + fn error_code(&self) -> Code { + self.inner.error_code() + } +} + +impl fmt::Display for ResponseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(f) + } +} + +impl From for ResponseError { + fn from(error: Error) -> ResponseError { + ResponseError { inner: Box::new(error) } + } +} + +impl From for ResponseError { + fn from(err: meilisearch_core::Error) -> ResponseError { + ResponseError { inner: Box::new(err) } + } +} + +impl From for ResponseError { + fn from(err: meilisearch_schema::Error) -> ResponseError { + ResponseError { inner: Box::new(err) } + } +} + +impl From for ResponseError { + fn from(err: FacetCountError) -> ResponseError { + ResponseError { inner: Box::new(err) } + } +} + +impl Serialize for ResponseError { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let struct_name = "ResponseError"; + let field_count = 4; + + let mut state = serializer.serialize_struct(struct_name, field_count)?; + state.serialize_field("message", &self.to_string())?; + state.serialize_field("errorCode", &self.error_name())?; + state.serialize_field("errorType", &self.error_type())?; + state.serialize_field("errorLink", &self.error_url())?; + state.end() + } +} + +impl aweb::error::ResponseError for ResponseError { + fn error_response(&self) -> aweb::HttpResponse { + ResponseBuilder::new(self.status_code()).json(&self) + } + + fn status_code(&self) -> StatusCode { + self.http_status() + } +} + +#[derive(Debug)] +pub enum Error { + BadParameter(String, String), + BadRequest(String), + CreateIndex(String), + DocumentNotFound(String), + IndexNotFound(String), + IndexAlreadyExists(String), + Internal(String), + InvalidIndexUid, + InvalidToken(String), + MissingAuthorizationHeader, + NotFound(String), + OpenIndex(String), + RetrieveDocument(u32, String), + SearchDocuments(String), + PayloadTooLarge, + UnsupportedMediaType, + DumpAlreadyInProgress, + DumpProcessFailed(String), +} + +impl error::Error for Error {} + +impl ErrorCode for Error { + fn error_code(&self) -> Code { + use Error::*; + match self { + BadParameter(_, _) => Code::BadParameter, + BadRequest(_) => Code::BadRequest, + CreateIndex(_) => Code::CreateIndex, + DocumentNotFound(_) => Code::DocumentNotFound, + IndexNotFound(_) => Code::IndexNotFound, + IndexAlreadyExists(_) => Code::IndexAlreadyExists, + Internal(_) => Code::Internal, + InvalidIndexUid => Code::InvalidIndexUid, + InvalidToken(_) => Code::InvalidToken, + MissingAuthorizationHeader => Code::MissingAuthorizationHeader, + NotFound(_) => Code::NotFound, + OpenIndex(_) => Code::OpenIndex, + RetrieveDocument(_, _) => Code::RetrieveDocument, + SearchDocuments(_) => Code::SearchDocuments, + PayloadTooLarge => Code::PayloadTooLarge, + UnsupportedMediaType => Code::UnsupportedMediaType, + DumpAlreadyInProgress => Code::DumpAlreadyInProgress, + DumpProcessFailed(_) => Code::DumpProcessFailed, + } + } +} + +#[derive(Debug)] +pub enum FacetCountError { + AttributeNotSet(String), + SyntaxError(String), + UnexpectedToken { found: String, expected: &'static [&'static str] }, + NoFacetSet, +} + +impl error::Error for FacetCountError {} + +impl ErrorCode for FacetCountError { + fn error_code(&self) -> Code { + Code::BadRequest + } +} + +impl FacetCountError { + pub fn unexpected_token(found: impl ToString, expected: &'static [&'static str]) -> FacetCountError { + let found = found.to_string(); + FacetCountError::UnexpectedToken { expected, found } + } +} + +impl From for FacetCountError { + fn from(other: serde_json::error::Error) -> FacetCountError { + FacetCountError::SyntaxError(other.to_string()) + } +} + +impl fmt::Display for FacetCountError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use FacetCountError::*; + + match self { + AttributeNotSet(attr) => write!(f, "Attribute {} is not set as facet", attr), + SyntaxError(msg) => write!(f, "Syntax error: {}", msg), + UnexpectedToken { expected, found } => write!(f, "Unexpected {} found, expected {:?}", found, expected), + NoFacetSet => write!(f, "Can't perform facet count, as no facet is set"), + } + } +} + +impl Error { + pub fn internal(err: impl fmt::Display) -> Error { + Error::Internal(err.to_string()) + } + + pub fn bad_request(err: impl fmt::Display) -> Error { + Error::BadRequest(err.to_string()) + } + + pub fn missing_authorization_header() -> Error { + Error::MissingAuthorizationHeader + } + + pub fn invalid_token(err: impl fmt::Display) -> Error { + Error::InvalidToken(err.to_string()) + } + + pub fn not_found(err: impl fmt::Display) -> Error { + Error::NotFound(err.to_string()) + } + + pub fn index_not_found(err: impl fmt::Display) -> Error { + Error::IndexNotFound(err.to_string()) + } + + pub fn document_not_found(err: impl fmt::Display) -> Error { + Error::DocumentNotFound(err.to_string()) + } + + pub fn bad_parameter(param: impl fmt::Display, err: impl fmt::Display) -> Error { + Error::BadParameter(param.to_string(), err.to_string()) + } + + pub fn open_index(err: impl fmt::Display) -> Error { + Error::OpenIndex(err.to_string()) + } + + pub fn create_index(err: impl fmt::Display) -> Error { + Error::CreateIndex(err.to_string()) + } + + pub fn invalid_index_uid() -> Error { + Error::InvalidIndexUid + } + + pub fn retrieve_document(doc_id: u32, err: impl fmt::Display) -> Error { + Error::RetrieveDocument(doc_id, err.to_string()) + } + + pub fn search_documents(err: impl fmt::Display) -> Error { + Error::SearchDocuments(err.to_string()) + } + + pub fn dump_conflict() -> Error { + Error::DumpAlreadyInProgress + } + + pub fn dump_failed(message: String) -> Error { + Error::DumpProcessFailed(message) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::BadParameter(param, err) => write!(f, "Url parameter {} error: {}", param, err), + Self::BadRequest(err) => f.write_str(err), + Self::CreateIndex(err) => write!(f, "Impossible to create index; {}", err), + Self::DocumentNotFound(document_id) => write!(f, "Document with id {} not found", document_id), + Self::IndexNotFound(index_uid) => write!(f, "Index {} not found", index_uid), + Self::IndexAlreadyExists(index_uid) => write!(f, "Index {} already exists", index_uid), + Self::Internal(err) => f.write_str(err), + Self::InvalidIndexUid => f.write_str("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."), + Self::InvalidToken(err) => write!(f, "Invalid API key: {}", err), + Self::MissingAuthorizationHeader => f.write_str("You must have an authorization token"), + Self::NotFound(err) => write!(f, "{} not found", err), + Self::OpenIndex(err) => write!(f, "Impossible to open index; {}", err), + Self::RetrieveDocument(id, err) => write!(f, "Impossible to retrieve the document with id: {}; {}", id, err), + Self::SearchDocuments(err) => write!(f, "Impossible to search documents; {}", err), + Self::PayloadTooLarge => f.write_str("Payload too large"), + Self::UnsupportedMediaType => f.write_str("Unsupported media type"), + Self::DumpAlreadyInProgress => f.write_str("Another dump is already in progress"), + Self::DumpProcessFailed(message) => write!(f, "Dump process failed: {}", message), + } + } +} + +impl From for Error { + fn from(err: std::io::Error) -> Error { + Error::Internal(err.to_string()) + } +} + +impl From for Error { + fn from(err: actix_http::Error) -> Error { + Error::Internal(err.to_string()) + } +} + +impl From for Error { + fn from(err: meilisearch_core::Error) -> Error { + Error::Internal(err.to_string()) + } +} + +impl From for Error { + fn from(err: serde_json::error::Error) -> Error { + Error::Internal(err.to_string()) + } +} + +impl From for Error { + fn from(err: JsonPayloadError) -> Error { + match err { + JsonPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid JSON: {}", err)), + JsonPayloadError::Overflow => Error::PayloadTooLarge, + JsonPayloadError::ContentType => Error::UnsupportedMediaType, + JsonPayloadError::Payload(err) => Error::BadRequest(format!("Problem while decoding the request: {}", err)), + } + } +} + +impl From for Error { + fn from(err: QueryPayloadError) -> Error { + match err { + QueryPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid query parameters: {}", err)), + } + } +} + +pub fn payload_error_handler>(err: E) -> ResponseError { + let error: Error = err.into(); + error.into() +} diff --git a/src/helpers/authentication.rs b/src/helpers/authentication.rs new file mode 100644 index 000000000..974c622f0 --- /dev/null +++ b/src/helpers/authentication.rs @@ -0,0 +1,103 @@ +use std::cell::RefCell; +use std::pin::Pin; +use std::rc::Rc; +use std::task::{Context, Poll}; + +use actix_service::{Service, Transform}; +use actix_web::{dev::ServiceRequest, dev::ServiceResponse, web}; +use futures::future::{err, ok, Future, Ready}; + +use crate::error::{Error, ResponseError}; +use crate::Data; + +#[derive(Clone)] +pub enum Authentication { + Public, + Private, + Admin, +} + +impl Transform for Authentication +where + S: Service, Error = actix_web::Error>, + S::Future: 'static, + B: 'static, +{ + type Request = ServiceRequest; + type Response = ServiceResponse; + type Error = actix_web::Error; + type InitError = (); + type Transform = LoggingMiddleware; + type Future = Ready>; + + fn new_transform(&self, service: S) -> Self::Future { + ok(LoggingMiddleware { + acl: self.clone(), + service: Rc::new(RefCell::new(service)), + }) + } +} + +pub struct LoggingMiddleware { + acl: Authentication, + service: Rc>, +} + +#[allow(clippy::type_complexity)] +impl Service for LoggingMiddleware +where + S: Service, Error = actix_web::Error> + 'static, + S::Future: 'static, + B: 'static, +{ + type Request = ServiceRequest; + type Response = ServiceResponse; + type Error = actix_web::Error; + type Future = Pin>>>; + + fn poll_ready(&mut self, cx: &mut Context) -> Poll> { + self.service.poll_ready(cx) + } + + fn call(&mut self, req: ServiceRequest) -> Self::Future { + let mut svc = self.service.clone(); + // This unwrap is left because this error should never appear. If that's the case, then + // it means that actix-web has an issue or someone changes the type `Data`. + let data = req.app_data::>().unwrap(); + + if data.api_keys.master.is_none() { + return Box::pin(svc.call(req)); + } + + let auth_header = match req.headers().get("X-Meili-API-Key") { + Some(auth) => match auth.to_str() { + Ok(auth) => auth, + Err(_) => return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())), + }, + None => { + return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())); + } + }; + + let authenticated = match self.acl { + Authentication::Admin => data.api_keys.master.as_deref() == Some(auth_header), + Authentication::Private => { + data.api_keys.master.as_deref() == Some(auth_header) + || data.api_keys.private.as_deref() == Some(auth_header) + } + Authentication::Public => { + data.api_keys.master.as_deref() == Some(auth_header) + || data.api_keys.private.as_deref() == Some(auth_header) + || data.api_keys.public.as_deref() == Some(auth_header) + } + }; + + if authenticated { + Box::pin(svc.call(req)) + } else { + Box::pin(err( + ResponseError::from(Error::InvalidToken(auth_header.to_string())).into() + )) + } + } +} diff --git a/src/helpers/compression.rs b/src/helpers/compression.rs new file mode 100644 index 000000000..ff3e1258f --- /dev/null +++ b/src/helpers/compression.rs @@ -0,0 +1,27 @@ +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use std::fs::{create_dir_all, File}; +use std::path::Path; +use tar::{Builder, Archive}; + +use crate::error::Error; + +pub fn to_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> { + let f = File::create(dest)?; + let gz_encoder = GzEncoder::new(f, Compression::default()); + let mut tar_encoder = Builder::new(gz_encoder); + tar_encoder.append_dir_all(".", src)?; + let gz_encoder = tar_encoder.into_inner()?; + gz_encoder.finish()?; + Ok(()) +} + +pub fn from_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> { + let f = File::open(src)?; + let gz = GzDecoder::new(f); + let mut ar = Archive::new(gz); + create_dir_all(dest)?; + ar.unpack(dest)?; + Ok(()) +} diff --git a/src/helpers/meilisearch.rs b/src/helpers/meilisearch.rs new file mode 100644 index 000000000..749fe410e --- /dev/null +++ b/src/helpers/meilisearch.rs @@ -0,0 +1,652 @@ +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; +use std::hash::{Hash, Hasher}; +use std::time::Instant; + +use indexmap::IndexMap; +use log::error; +use meilisearch_core::{Filter, MainReader}; +use meilisearch_core::facets::FacetFilter; +use meilisearch_core::criterion::*; +use meilisearch_core::settings::RankingRule; +use meilisearch_core::{Highlight, Index, RankedMap}; +use meilisearch_schema::{FieldId, Schema}; +use meilisearch_tokenizer::is_cjk; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use siphasher::sip::SipHasher; +use slice_group_by::GroupBy; + +use crate::error::{Error, ResponseError}; + +pub trait IndexSearchExt { + fn new_search(&self, query: Option) -> SearchBuilder; +} + +impl IndexSearchExt for Index { + fn new_search(&self, query: Option) -> SearchBuilder { + SearchBuilder { + index: self, + query, + offset: 0, + limit: 20, + attributes_to_crop: None, + attributes_to_retrieve: None, + attributes_to_highlight: None, + filters: None, + matches: false, + facet_filters: None, + facets: None, + } + } +} + +pub struct SearchBuilder<'a> { + index: &'a Index, + query: Option, + offset: usize, + limit: usize, + attributes_to_crop: Option>, + attributes_to_retrieve: Option>, + attributes_to_highlight: Option>, + filters: Option, + matches: bool, + facet_filters: Option, + facets: Option> +} + +impl<'a> SearchBuilder<'a> { + pub fn offset(&mut self, value: usize) -> &SearchBuilder { + self.offset = value; + self + } + + pub fn limit(&mut self, value: usize) -> &SearchBuilder { + self.limit = value; + self + } + + pub fn attributes_to_crop(&mut self, value: HashMap) -> &SearchBuilder { + self.attributes_to_crop = Some(value); + self + } + + pub fn attributes_to_retrieve(&mut self, value: HashSet) -> &SearchBuilder { + self.attributes_to_retrieve = Some(value); + self + } + + pub fn add_retrievable_field(&mut self, value: String) -> &SearchBuilder { + let attributes_to_retrieve = self.attributes_to_retrieve.get_or_insert(HashSet::new()); + attributes_to_retrieve.insert(value); + self + } + + pub fn attributes_to_highlight(&mut self, value: HashSet) -> &SearchBuilder { + self.attributes_to_highlight = Some(value); + self + } + + pub fn add_facet_filters(&mut self, filters: FacetFilter) -> &SearchBuilder { + self.facet_filters = Some(filters); + self + } + + pub fn filters(&mut self, value: String) -> &SearchBuilder { + self.filters = Some(value); + self + } + + pub fn get_matches(&mut self) -> &SearchBuilder { + self.matches = true; + self + } + + pub fn add_facets(&mut self, facets: Vec<(FieldId, String)>) -> &SearchBuilder { + self.facets = Some(facets); + self + } + + pub fn search(self, reader: &MainReader) -> Result { + let schema = self + .index + .main + .schema(reader)? + .ok_or(Error::internal("missing schema"))?; + + let ranked_map = self.index.main.ranked_map(reader)?.unwrap_or_default(); + + // Change criteria + let mut query_builder = match self.get_criteria(reader, &ranked_map, &schema)? { + Some(criteria) => self.index.query_builder_with_criteria(criteria), + None => self.index.query_builder(), + }; + + if let Some(filter_expression) = &self.filters { + let filter = Filter::parse(filter_expression, &schema)?; + let index = &self.index; + query_builder.with_filter(move |id| { + let reader = &reader; + let filter = &filter; + match filter.test(reader, index, id) { + Ok(res) => res, + Err(e) => { + log::warn!("unexpected error during filtering: {}", e); + false + } + } + }); + } + + if let Some(field) = self.index.main.distinct_attribute(reader)? { + let index = &self.index; + query_builder.with_distinct(1, move |id| { + match index.document_attribute_bytes(reader, id, field) { + Ok(Some(bytes)) => { + let mut s = SipHasher::new(); + bytes.hash(&mut s); + Some(s.finish()) + } + _ => None, + } + }); + } + + query_builder.set_facet_filter(self.facet_filters); + query_builder.set_facets(self.facets); + + let start = Instant::now(); + let result = query_builder.query(reader, self.query.as_deref(), self.offset..(self.offset + self.limit)); + let search_result = result.map_err(Error::search_documents)?; + let time_ms = start.elapsed().as_millis() as usize; + + let mut all_attributes: HashSet<&str> = HashSet::new(); + let mut all_formatted: HashSet<&str> = HashSet::new(); + + match &self.attributes_to_retrieve { + Some(to_retrieve) => { + all_attributes.extend(to_retrieve.iter().map(String::as_str)); + + if let Some(to_highlight) = &self.attributes_to_highlight { + all_formatted.extend(to_highlight.iter().map(String::as_str)); + } + + if let Some(to_crop) = &self.attributes_to_crop { + all_formatted.extend(to_crop.keys().map(String::as_str)); + } + + all_attributes.extend(&all_formatted); + }, + None => { + all_attributes.extend(schema.displayed_name()); + // If we specified at least one attribute to highlight or crop then + // all available attributes will be returned in the _formatted field. + if self.attributes_to_highlight.is_some() || self.attributes_to_crop.is_some() { + all_formatted.extend(all_attributes.iter().cloned()); + } + }, + } + + let mut hits = Vec::with_capacity(self.limit); + for doc in search_result.documents { + let mut document: IndexMap = self + .index + .document(reader, Some(&all_attributes), doc.id) + .map_err(|e| Error::retrieve_document(doc.id.0, e))? + .ok_or(Error::internal( + "Impossible to retrieve the document; Corrupted data", + ))?; + + let mut formatted = document.iter() + .filter(|(key, _)| all_formatted.contains(key.as_str())) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + + let mut matches = doc.highlights.clone(); + + // Crops fields if needed + if let Some(fields) = &self.attributes_to_crop { + crop_document(&mut formatted, &mut matches, &schema, fields); + } + + // Transform to readable matches + if let Some(attributes_to_highlight) = &self.attributes_to_highlight { + let matches = calculate_matches( + &matches, + self.attributes_to_highlight.clone(), + &schema, + ); + formatted = calculate_highlights(&formatted, &matches, attributes_to_highlight); + } + + let matches_info = if self.matches { + Some(calculate_matches(&matches, self.attributes_to_retrieve.clone(), &schema)) + } else { + None + }; + + if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve { + document.retain(|key, _| attributes_to_retrieve.contains(&key.to_string())) + } + + let hit = SearchHit { + document, + formatted, + matches_info, + }; + + hits.push(hit); + } + + let results = SearchResult { + hits, + offset: self.offset, + limit: self.limit, + nb_hits: search_result.nb_hits, + exhaustive_nb_hits: search_result.exhaustive_nb_hit, + processing_time_ms: time_ms, + query: self.query.unwrap_or_default(), + facets_distribution: search_result.facets, + exhaustive_facets_count: search_result.exhaustive_facets_count, + }; + + Ok(results) + } + + pub fn get_criteria( + &self, + reader: &MainReader, + ranked_map: &'a RankedMap, + schema: &Schema, + ) -> Result>, ResponseError> { + let ranking_rules = self.index.main.ranking_rules(reader)?; + + if let Some(ranking_rules) = ranking_rules { + let mut builder = CriteriaBuilder::with_capacity(7 + ranking_rules.len()); + for rule in ranking_rules { + match rule { + RankingRule::Typo => builder.push(Typo), + RankingRule::Words => builder.push(Words), + RankingRule::Proximity => builder.push(Proximity), + RankingRule::Attribute => builder.push(Attribute), + RankingRule::WordsPosition => builder.push(WordsPosition), + RankingRule::Exactness => builder.push(Exactness), + RankingRule::Asc(field) => { + match SortByAttr::lower_is_better(&ranked_map, &schema, &field) { + Ok(rule) => builder.push(rule), + Err(err) => error!("Error during criteria builder; {:?}", err), + } + } + RankingRule::Desc(field) => { + match SortByAttr::higher_is_better(&ranked_map, &schema, &field) { + Ok(rule) => builder.push(rule), + Err(err) => error!("Error during criteria builder; {:?}", err), + } + } + } + } + builder.push(DocumentId); + return Ok(Some(builder.build())); + } + + Ok(None) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct MatchPosition { + pub start: usize, + pub length: usize, +} + +impl PartialOrd for MatchPosition { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MatchPosition { + fn cmp(&self, other: &Self) -> Ordering { + match self.start.cmp(&other.start) { + Ordering::Equal => self.length.cmp(&other.length), + _ => self.start.cmp(&other.start), + } + } +} + +pub type HighlightInfos = HashMap; +pub type MatchesInfos = HashMap>; +// pub type RankingInfos = HashMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchHit { + #[serde(flatten)] + pub document: IndexMap, + #[serde(rename = "_formatted", skip_serializing_if = "IndexMap::is_empty")] + pub formatted: IndexMap, + #[serde(rename = "_matchesInfo", skip_serializing_if = "Option::is_none")] + pub matches_info: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchResult { + pub hits: Vec, + pub offset: usize, + pub limit: usize, + pub nb_hits: usize, + pub exhaustive_nb_hits: bool, + pub processing_time_ms: usize, + pub query: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub facets_distribution: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub exhaustive_facets_count: Option, +} + +/// returns the start index and the length on the crop. +fn aligned_crop(text: &str, match_index: usize, context: usize) -> (usize, usize) { + let is_word_component = |c: &char| c.is_alphanumeric() && !is_cjk(*c); + + let word_end_index = |mut index| { + if text.chars().nth(index - 1).map_or(false, |c| is_word_component(&c)) { + index += text.chars().skip(index).take_while(is_word_component).count(); + } + index + }; + + if context == 0 { + // count need to be at least 1 for cjk queries to return something + return (match_index, 1 + text.chars().skip(match_index).take_while(is_word_component).count()); + } + let start = match match_index.saturating_sub(context) { + 0 => 0, + n => { + let word_end_index = word_end_index(n); + // skip whitespaces if any + word_end_index + text.chars().skip(word_end_index).take_while(char::is_ascii_whitespace).count() + } + }; + let end = word_end_index(match_index + context); + + (start, end - start) +} + +fn crop_text( + text: &str, + matches: impl IntoIterator, + context: usize, +) -> (String, Vec) { + let mut matches = matches.into_iter().peekable(); + + let char_index = matches.peek().map(|m| m.char_index as usize).unwrap_or(0); + let (start, count) = aligned_crop(text, char_index, context); + + // TODO do something about double allocation + let text = text + .chars() + .skip(start) + .take(count) + .collect::() + .trim() + .to_string(); + + // update matches index to match the new cropped text + let matches = matches + .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + count) + .map(|m| Highlight { + char_index: m.char_index - start as u16, + ..m + }) + .collect(); + + (text, matches) +} + +fn crop_document( + document: &mut IndexMap, + matches: &mut Vec, + schema: &Schema, + fields: &HashMap, +) { + matches.sort_unstable_by_key(|m| (m.char_index, m.char_length)); + + for (field, length) in fields { + let attribute = match schema.id(field) { + Some(attribute) => attribute, + None => continue, + }; + + let selected_matches = matches + .iter() + .filter(|m| FieldId::new(m.attribute) == attribute) + .cloned(); + + if let Some(Value::String(ref mut original_text)) = document.get_mut(field) { + let (cropped_text, cropped_matches) = + crop_text(original_text, selected_matches, *length); + + *original_text = cropped_text; + + matches.retain(|m| FieldId::new(m.attribute) != attribute); + matches.extend_from_slice(&cropped_matches); + } + } +} + +fn calculate_matches( + matches: &[Highlight], + attributes_to_retrieve: Option>, + schema: &Schema, +) -> MatchesInfos { + let mut matches_result: HashMap> = HashMap::new(); + for m in matches.iter() { + if let Some(attribute) = schema.name(FieldId::new(m.attribute)) { + if let Some(ref attributes_to_retrieve) = attributes_to_retrieve { + if !attributes_to_retrieve.contains(attribute) { + continue; + } + } + if !schema.displayed_name().contains(attribute) { + continue; + } + if let Some(pos) = matches_result.get_mut(attribute) { + pos.push(MatchPosition { + start: m.char_index as usize, + length: m.char_length as usize, + }); + } else { + let mut positions = Vec::new(); + positions.push(MatchPosition { + start: m.char_index as usize, + length: m.char_length as usize, + }); + matches_result.insert(attribute.to_string(), positions); + } + } + } + for (_, val) in matches_result.iter_mut() { + val.sort_unstable(); + val.dedup(); + } + matches_result +} + +fn calculate_highlights( + document: &IndexMap, + matches: &MatchesInfos, + attributes_to_highlight: &HashSet, +) -> IndexMap { + let mut highlight_result = document.clone(); + + for (attribute, matches) in matches.iter() { + if attributes_to_highlight.contains(attribute) { + if let Some(Value::String(value)) = document.get(attribute) { + let value: Vec<_> = value.chars().collect(); + let mut highlighted_value = String::new(); + let mut index = 0; + + let longest_matches = matches + .linear_group_by_key(|m| m.start) + .map(|group| group.last().unwrap()) + .filter(move |m| m.start >= index); + + for m in longest_matches { + let before = value.get(index..m.start); + let highlighted = value.get(m.start..(m.start + m.length)); + if let (Some(before), Some(highlighted)) = (before, highlighted) { + highlighted_value.extend(before); + highlighted_value.push_str(""); + highlighted_value.extend(highlighted); + highlighted_value.push_str(""); + index = m.start + m.length; + } else { + error!("value: {:?}; index: {:?}, match: {:?}", value, index, m); + } + } + highlighted_value.extend(value[index..].iter()); + highlight_result.insert(attribute.to_string(), Value::String(highlighted_value)); + }; + } + } + highlight_result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aligned_crops() { + let text = r#"En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation."#; + + // simple test + let (start, length) = aligned_crop(&text, 6, 2); + let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); + assert_eq!("début", cropped); + + // first word test + let (start, length) = aligned_crop(&text, 0, 1); + let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); + assert_eq!("En", cropped); + // last word test + let (start, length) = aligned_crop(&text, 510, 2); + let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); + assert_eq!("Fondation", cropped); + + // CJK tests + let text = "this isのス foo myタイリ test"; + + // mixed charset + let (start, length) = aligned_crop(&text, 5, 3); + let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); + assert_eq!("isの", cropped); + + // split regular word / CJK word, no space + let (start, length) = aligned_crop(&text, 7, 1); + let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); + assert_eq!("の", cropped); + } + + #[test] + fn calculate_matches() { + let mut matches = Vec::new(); + matches.push(Highlight { attribute: 0, char_index: 0, char_length: 3}); + matches.push(Highlight { attribute: 0, char_index: 0, char_length: 2}); + + let mut attributes_to_retrieve: HashSet = HashSet::new(); + attributes_to_retrieve.insert("title".to_string()); + + let schema = Schema::with_primary_key("title"); + + let matches_result = super::calculate_matches(&matches, Some(attributes_to_retrieve), &schema); + + let mut matches_result_expected: HashMap> = HashMap::new(); + + let mut positions = Vec::new(); + positions.push(MatchPosition { + start: 0, + length: 2, + }); + positions.push(MatchPosition { + start: 0, + length: 3, + }); + matches_result_expected.insert("title".to_string(), positions); + + assert_eq!(matches_result, matches_result_expected); + } + + #[test] + fn calculate_highlights() { + let data = r#"{ + "title": "Fondation (Isaac ASIMOV)", + "description": "En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation." + }"#; + + let document: IndexMap = serde_json::from_str(data).unwrap(); + let mut attributes_to_highlight = HashSet::new(); + attributes_to_highlight.insert("title".to_string()); + attributes_to_highlight.insert("description".to_string()); + + let mut matches = HashMap::new(); + + let mut m = Vec::new(); + m.push(MatchPosition { + start: 0, + length: 9, + }); + matches.insert("title".to_string(), m); + + let mut m = Vec::new(); + m.push(MatchPosition { + start: 510, + length: 9, + }); + matches.insert("description".to_string(), m); + let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight); + + let mut result_expected = IndexMap::new(); + result_expected.insert( + "title".to_string(), + Value::String("Fondation (Isaac ASIMOV)".to_string()), + ); + result_expected.insert("description".to_string(), Value::String("En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation.".to_string())); + + assert_eq!(result, result_expected); + } + + #[test] + fn highlight_longest_match() { + let data = r#"{ + "title": "Ice" + }"#; + + let document: IndexMap = serde_json::from_str(data).unwrap(); + let mut attributes_to_highlight = HashSet::new(); + attributes_to_highlight.insert("title".to_string()); + + let mut matches = HashMap::new(); + + let mut m = Vec::new(); + m.push(MatchPosition { + start: 0, + length: 2, + }); + m.push(MatchPosition { + start: 0, + length: 3, + }); + matches.insert("title".to_string(), m); + + let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight); + + let mut result_expected = IndexMap::new(); + result_expected.insert( + "title".to_string(), + Value::String("Ice".to_string()), + ); + + assert_eq!(result, result_expected); + } +} diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs new file mode 100644 index 000000000..471336db9 --- /dev/null +++ b/src/helpers/mod.rs @@ -0,0 +1,7 @@ +pub mod authentication; +pub mod meilisearch; +pub mod normalize_path; +pub mod compression; + +pub use authentication::Authentication; +pub use normalize_path::NormalizePath; diff --git a/src/helpers/normalize_path.rs b/src/helpers/normalize_path.rs new file mode 100644 index 000000000..e669b9d94 --- /dev/null +++ b/src/helpers/normalize_path.rs @@ -0,0 +1,86 @@ +/// From https://docs.rs/actix-web/3.0.0-alpha.2/src/actix_web/middleware/normalize.rs.html#34 +use actix_http::Error; +use actix_service::{Service, Transform}; +use actix_web::{ + dev::ServiceRequest, + dev::ServiceResponse, + http::uri::{PathAndQuery, Uri}, +}; +use futures::future::{ok, Ready}; +use regex::Regex; +use std::task::{Context, Poll}; +pub struct NormalizePath; + +impl Transform for NormalizePath +where + S: Service, Error = Error>, + S::Future: 'static, +{ + type Request = ServiceRequest; + type Response = ServiceResponse; + type Error = Error; + type InitError = (); + type Transform = NormalizePathNormalization; + type Future = Ready>; + + fn new_transform(&self, service: S) -> Self::Future { + ok(NormalizePathNormalization { + service, + merge_slash: Regex::new("//+").unwrap(), + }) + } +} + +pub struct NormalizePathNormalization { + service: S, + merge_slash: Regex, +} + +impl Service for NormalizePathNormalization +where + S: Service, Error = Error>, + S::Future: 'static, +{ + type Request = ServiceRequest; + type Response = ServiceResponse; + type Error = Error; + type Future = S::Future; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.service.poll_ready(cx) + } + + fn call(&mut self, mut req: ServiceRequest) -> Self::Future { + let head = req.head_mut(); + + // always add trailing slash, might be an extra one + let path = head.uri.path().to_string() + "/"; + + if self.merge_slash.find(&path).is_some() { + // normalize multiple /'s to one / + let path = self.merge_slash.replace_all(&path, "/"); + + let path = if path.len() > 1 { + path.trim_end_matches('/') + } else { + &path + }; + + let mut parts = head.uri.clone().into_parts(); + let pq = parts.path_and_query.as_ref().unwrap(); + + let path = if let Some(q) = pq.query() { + bytes::Bytes::from(format!("{}?{}", path, q)) + } else { + bytes::Bytes::copy_from_slice(path.as_bytes()) + }; + parts.path_and_query = Some(PathAndQuery::from_maybe_shared(path).unwrap()); + + let uri = Uri::from_parts(parts).unwrap(); + req.match_info_mut().get_mut().update(&uri); + req.head_mut().uri = uri; + } + + self.service.call(req) + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 000000000..b5f35f277 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,104 @@ +#![allow(clippy::or_fun_call)] + +pub mod data; +pub mod error; +pub mod helpers; +pub mod models; +pub mod option; +pub mod routes; +pub mod analytics; +pub mod snapshot; +pub mod dump; + +use actix_http::Error; +use actix_service::ServiceFactory; +use actix_web::{dev, web, App}; +use chrono::Utc; +use log::error; + +use meilisearch_core::{Index, MainWriter, ProcessedUpdateResult}; + +pub use option::Opt; +pub use self::data::Data; +use self::error::{payload_error_handler, ResponseError}; + +pub fn create_app( + data: &Data, + enable_frontend: bool, +) -> App< + impl ServiceFactory< + Config = (), + Request = dev::ServiceRequest, + Response = dev::ServiceResponse, + Error = Error, + InitError = (), + >, + actix_http::body::Body, +> { + let app = App::new() + .data(data.clone()) + .app_data( + web::JsonConfig::default() + .limit(data.http_payload_size_limit) + .content_type(|_mime| true) // Accept all mime types + .error_handler(|err, _req| payload_error_handler(err).into()), + ) + .app_data( + web::QueryConfig::default() + .error_handler(|err, _req| payload_error_handler(err).into()) + ) + .configure(routes::document::services) + .configure(routes::index::services) + .configure(routes::search::services) + .configure(routes::setting::services) + .configure(routes::stop_words::services) + .configure(routes::synonym::services) + .configure(routes::health::services) + .configure(routes::stats::services) + .configure(routes::key::services) + .configure(routes::dump::services); + if enable_frontend { + app + .service(routes::load_html) + .service(routes::load_css) + } else { + app + } +} + +pub fn index_update_callback_txn(index: Index, index_uid: &str, data: &Data, mut writer: &mut MainWriter) -> Result<(), String> { + if let Err(e) = data.db.compute_stats(&mut writer, index_uid) { + return Err(format!("Impossible to compute stats; {}", e)); + } + + if let Err(e) = data.db.set_last_update(&mut writer, &Utc::now()) { + return Err(format!("Impossible to update last_update; {}", e)); + } + + if let Err(e) = index.main.put_updated_at(&mut writer) { + return Err(format!("Impossible to update updated_at; {}", e)); + } + + Ok(()) +} + +pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) { + if status.error.is_some() { + return; + } + + if let Some(index) = data.db.open_index(index_uid) { + let db = &data.db; + let res = db.main_write::<_, _, ResponseError>(|mut writer| { + if let Err(e) = index_update_callback_txn(index, index_uid, data, &mut writer) { + error!("{}", e); + } + + Ok(()) + }); + match res { + Ok(_) => (), + Err(e) => error!("{}", e), + } + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 000000000..800e2760c --- /dev/null +++ b/src/main.rs @@ -0,0 +1,169 @@ +use std::{env, thread}; + +use actix_cors::Cors; +use actix_web::{middleware, HttpServer}; +use main_error::MainError; +use meilisearch_http::helpers::NormalizePath; +use meilisearch_http::{create_app, index_update_callback, Data, Opt}; +use structopt::StructOpt; +use meilisearch_http::{snapshot, dump}; + +mod analytics; + +#[cfg(target_os = "linux")] +#[global_allocator] +static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + +#[actix_web::main] +async fn main() -> Result<(), MainError> { + let opt = Opt::from_args(); + + #[cfg(all(not(debug_assertions), feature = "sentry"))] + let _sentry = sentry::init(( + if !opt.no_sentry { + Some(opt.sentry_dsn.clone()) + } else { + None + }, + sentry::ClientOptions { + release: sentry::release_name!(), + ..Default::default() + }, + )); + + match opt.env.as_ref() { + "production" => { + if opt.master_key.is_none() { + return Err( + "In production mode, the environment variable MEILI_MASTER_KEY is mandatory" + .into(), + ); + } + + #[cfg(all(not(debug_assertions), feature = "sentry"))] + if !opt.no_sentry && _sentry.is_enabled() { + sentry::integrations::panic::register_panic_handler(); // TODO: This shouldn't be needed when upgrading to sentry 0.19.0. These integrations are turned on by default when using `sentry::init`. + sentry::integrations::env_logger::init(None, Default::default()); + } + } + "development" => { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + } + _ => unreachable!(), + } + + if let Some(path) = &opt.import_snapshot { + snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; + } + + let data = Data::new(opt.clone())?; + + if !opt.no_analytics { + let analytics_data = data.clone(); + let analytics_opt = opt.clone(); + thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); + } + + let data_cloned = data.clone(); + data.db.set_update_callback(Box::new(move |name, status| { + index_update_callback(name, &data_cloned, status); + })); + + + if let Some(path) = &opt.import_dump { + dump::import_dump(&data, path, opt.dump_batch_size)?; + } + + if opt.schedule_snapshot { + snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; + } + + print_launch_resume(&opt, &data); + + let enable_frontend = opt.env != "production"; + let http_server = HttpServer::new(move || { + create_app(&data, enable_frontend) + .wrap( + Cors::default() + .send_wildcard() + .allowed_headers(vec!["content-type", "x-meili-api-key"]) + .max_age(86_400) // 24h + ) + .wrap(middleware::Logger::default()) + .wrap(middleware::Compress::default()) + .wrap(NormalizePath) + }); + + if let Some(config) = opt.get_ssl_config()? { + http_server + .bind_rustls(opt.http_addr, config)? + .run() + .await?; + } else { + http_server.bind(opt.http_addr)?.run().await?; + } + + Ok(()) +} + +pub fn print_launch_resume(opt: &Opt, data: &Data) { + let ascii_name = r#" +888b d888 d8b 888 d8b .d8888b. 888 +8888b d8888 Y8P 888 Y8P d88P Y88b 888 +88888b.d88888 888 Y88b. 888 +888Y88888P888 .d88b. 888 888 888 "Y888b. .d88b. 8888b. 888d888 .d8888b 88888b. +888 Y888P 888 d8P Y8b 888 888 888 "Y88b. d8P Y8b "88b 888P" d88P" 888 "88b +888 Y8P 888 88888888 888 888 888 "888 88888888 .d888888 888 888 888 888 +888 " 888 Y8b. 888 888 888 Y88b d88P Y8b. 888 888 888 Y88b. 888 888 +888 888 "Y8888 888 888 888 "Y8888P" "Y8888 "Y888888 888 "Y8888P 888 888 +"#; + + eprintln!("{}", ascii_name); + + eprintln!("Database path:\t\t{:?}", opt.db_path); + eprintln!("Server listening on:\t{:?}", opt.http_addr); + eprintln!("Environment:\t\t{:?}", opt.env); + eprintln!("Commit SHA:\t\t{:?}", env!("VERGEN_SHA").to_string()); + eprintln!( + "Build date:\t\t{:?}", + env!("VERGEN_BUILD_TIMESTAMP").to_string() + ); + eprintln!( + "Package version:\t{:?}", + env!("CARGO_PKG_VERSION").to_string() + ); + + #[cfg(all(not(debug_assertions), feature = "sentry"))] + eprintln!( + "Sentry DSN:\t\t{:?}", + if !opt.no_sentry { + &opt.sentry_dsn + } else { + "Disabled" + } + ); + + eprintln!( + "Amplitude Analytics:\t{:?}", + if !opt.no_analytics { + "Enabled" + } else { + "Disabled" + } + ); + + eprintln!(); + + if data.api_keys.master.is_some() { + eprintln!("A Master Key has been set. Requests to MeiliSearch won't be authorized unless you provide an authentication key."); + } else { + eprintln!("No master key found; The server will accept unidentified requests. \ + If you need some protection in development mode, please export a key: export MEILI_MASTER_KEY=xxx"); + } + + eprintln!(); + eprintln!("Documentation:\t\thttps://docs.meilisearch.com"); + eprintln!("Source code:\t\thttps://github.com/meilisearch/meilisearch"); + eprintln!("Contact:\t\thttps://docs.meilisearch.com/resources/contact.html or bonjour@meilisearch.com"); + eprintln!(); +} diff --git a/src/models/mod.rs b/src/models/mod.rs new file mode 100644 index 000000000..82e7e77c4 --- /dev/null +++ b/src/models/mod.rs @@ -0,0 +1 @@ +pub mod update_operation; diff --git a/src/models/update_operation.rs b/src/models/update_operation.rs new file mode 100644 index 000000000..e7a41b10b --- /dev/null +++ b/src/models/update_operation.rs @@ -0,0 +1,33 @@ +use std::fmt; + +#[allow(dead_code)] +#[derive(Debug)] +pub enum UpdateOperation { + ClearAllDocuments, + DocumentsAddition, + DocumentsDeletion, + SynonymsUpdate, + SynonymsDeletion, + StopWordsAddition, + StopWordsDeletion, + Schema, + Config, +} + +impl fmt::Display for UpdateOperation { + fn fmt(&self, f: &mut fmt::Formatter) -> std::fmt::Result { + use UpdateOperation::*; + + match self { + ClearAllDocuments => write!(f, "ClearAllDocuments"), + DocumentsAddition => write!(f, "DocumentsAddition"), + DocumentsDeletion => write!(f, "DocumentsDeletion"), + SynonymsUpdate => write!(f, "SynonymsUpdate"), + SynonymsDeletion => write!(f, "SynonymsDelettion"), + StopWordsAddition => write!(f, "StopWordsAddition"), + StopWordsDeletion => write!(f, "StopWordsDeletion"), + Schema => write!(f, "Schema"), + Config => write!(f, "Config"), + } + } +} diff --git a/src/option.rs b/src/option.rs new file mode 100644 index 000000000..e1d74fd63 --- /dev/null +++ b/src/option.rs @@ -0,0 +1,221 @@ +use std::{error, fs}; +use std::io::{BufReader, Read}; +use std::path::PathBuf; +use std::sync::Arc; + +use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; +use rustls::{ + AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, + RootCertStore, +}; +use structopt::StructOpt; + +const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; + +#[derive(Debug, Default, Clone, StructOpt)] +pub struct Opt { + /// The destination where the database must be created. + #[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")] + pub db_path: String, + + /// The address on which the http server will listen. + #[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")] + pub http_addr: String, + + /// The master key allowing you to do everything on the server. + #[structopt(long, env = "MEILI_MASTER_KEY")] + pub master_key: Option, + + /// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project. + /// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable. + #[cfg(all(not(debug_assertions), feature = "sentry"))] + #[structopt(long, env = "SENTRY_DSN", default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337")] + pub sentry_dsn: String, + + /// Disable Sentry error reporting. + #[cfg(all(not(debug_assertions), feature = "sentry"))] + #[structopt(long, env = "MEILI_NO_SENTRY")] + pub no_sentry: bool, + + /// This environment variable must be set to `production` if you are running in production. + /// If the server is running in development mode more logs will be displayed, + /// and the master key can be avoided which implies that there is no security on the updates routes. + /// This is useful to debug when integrating the engine with another service. + #[structopt(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)] + pub env: String, + + /// Do not send analytics to Meili. + #[structopt(long, env = "MEILI_NO_ANALYTICS")] + pub no_analytics: bool, + + /// The maximum size, in bytes, of the main lmdb database directory + #[structopt(long, env = "MEILI_MAX_MDB_SIZE", default_value = "107374182400")] // 100GB + pub max_mdb_size: usize, + + /// The maximum size, in bytes, of the update lmdb database directory + #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "107374182400")] // 100GB + pub max_udb_size: usize, + + /// The maximum size, in bytes, of accepted JSON payloads + #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "10485760")] // 10MB + pub http_payload_size_limit: usize, + + /// Read server certificates from CERTFILE. + /// This should contain PEM-format certificates + /// in the right order (the first certificate should + /// certify KEYFILE, the last should be a root CA). + #[structopt(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))] + pub ssl_cert_path: Option, + + /// Read private key from KEYFILE. This should be a RSA + /// private key or PKCS8-encoded private key, in PEM format. + #[structopt(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))] + pub ssl_key_path: Option, + + /// Enable client authentication, and accept certificates + /// signed by those roots provided in CERTFILE. + #[structopt(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))] + pub ssl_auth_path: Option, + + /// Read DER-encoded OCSP response from OCSPFILE and staple to certificate. + /// Optional + #[structopt(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))] + pub ssl_ocsp_path: Option, + + /// Send a fatal alert if the client does not complete client authentication. + #[structopt(long, env = "MEILI_SSL_REQUIRE_AUTH")] + pub ssl_require_auth: bool, + + /// SSL support session resumption + #[structopt(long, env = "MEILI_SSL_RESUMPTION")] + pub ssl_resumption: bool, + + /// SSL support tickets. + #[structopt(long, env = "MEILI_SSL_TICKETS")] + pub ssl_tickets: bool, + + /// Defines the path of the snapshot file to import. + /// This option will, by default, stop the process if a database already exist or if no snapshot exists at + /// the given path. If this option is not specified no snapshot is imported. + #[structopt(long)] + pub import_snapshot: Option, + + /// The engine will ignore a missing snapshot and not return an error in such case. + #[structopt(long, requires = "import-snapshot")] + pub ignore_missing_snapshot: bool, + + /// The engine will skip snapshot importation and not return an error in such case. + #[structopt(long, requires = "import-snapshot")] + pub ignore_snapshot_if_db_exists: bool, + + /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap. + #[structopt(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")] + pub snapshot_dir: PathBuf, + + /// Activate snapshot scheduling. + #[structopt(long, env = "MEILI_SCHEDULE_SNAPSHOT")] + pub schedule_snapshot: bool, + + /// Defines time interval, in seconds, between each snapshot creation. + #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC")] + pub snapshot_interval_sec: Option, + + /// Folder where dumps are created when the dump route is called. + #[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")] + pub dumps_dir: PathBuf, + + /// Import a dump from the specified path, must be a `.tar.gz` file. + #[structopt(long, conflicts_with = "import-snapshot")] + pub import_dump: Option, + + /// The batch size used in the importation process, the bigger it is the faster the dump is created. + #[structopt(long, env = "MEILI_DUMP_BATCH_SIZE", default_value = "1024")] + pub dump_batch_size: usize, +} + +impl Opt { + pub fn get_ssl_config(&self) -> Result, Box> { + if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) { + let client_auth = match &self.ssl_auth_path { + Some(auth_path) => { + let roots = load_certs(auth_path.to_path_buf())?; + let mut client_auth_roots = RootCertStore::empty(); + for root in roots { + client_auth_roots.add(&root).unwrap(); + } + if self.ssl_require_auth { + AllowAnyAuthenticatedClient::new(client_auth_roots) + } else { + AllowAnyAnonymousOrAuthenticatedClient::new(client_auth_roots) + } + } + None => NoClientAuth::new(), + }; + + let mut config = rustls::ServerConfig::new(client_auth); + config.key_log = Arc::new(rustls::KeyLogFile::new()); + + let certs = load_certs(cert_path.to_path_buf())?; + let privkey = load_private_key(key_path.to_path_buf())?; + let ocsp = load_ocsp(&self.ssl_ocsp_path)?; + config + .set_single_cert_with_ocsp_and_sct(certs, privkey, ocsp, vec![]) + .map_err(|_| "bad certificates/private key")?; + + if self.ssl_resumption { + config.set_persistence(rustls::ServerSessionMemoryCache::new(256)); + } + + if self.ssl_tickets { + config.ticketer = rustls::Ticketer::new(); + } + + Ok(Some(config)) + } else { + Ok(None) + } + } +} + +fn load_certs(filename: PathBuf) -> Result, Box> { + let certfile = fs::File::open(filename).map_err(|_| "cannot open certificate file")?; + let mut reader = BufReader::new(certfile); + Ok(certs(&mut reader).map_err(|_| "cannot read certificate file")?) +} + +fn load_private_key(filename: PathBuf) -> Result> { + let rsa_keys = { + let keyfile = + fs::File::open(filename.clone()).map_err(|_| "cannot open private key file")?; + let mut reader = BufReader::new(keyfile); + rsa_private_keys(&mut reader).map_err(|_| "file contains invalid rsa private key")? + }; + + let pkcs8_keys = { + let keyfile = fs::File::open(filename).map_err(|_| "cannot open private key file")?; + let mut reader = BufReader::new(keyfile); + pkcs8_private_keys(&mut reader) + .map_err(|_| "file contains invalid pkcs8 private key (encrypted keys not supported)")? + }; + + // prefer to load pkcs8 keys + if !pkcs8_keys.is_empty() { + Ok(pkcs8_keys[0].clone()) + } else { + assert!(!rsa_keys.is_empty()); + Ok(rsa_keys[0].clone()) + } +} + +fn load_ocsp(filename: &Option) -> Result, Box> { + let mut ret = Vec::new(); + + if let Some(ref name) = filename { + fs::File::open(name) + .map_err(|_| "cannot open ocsp file")? + .read_to_end(&mut ret) + .map_err(|_| "cannot read oscp file")?; + } + + Ok(ret) +} diff --git a/src/routes/document.rs b/src/routes/document.rs new file mode 100644 index 000000000..e8fa0f646 --- /dev/null +++ b/src/routes/document.rs @@ -0,0 +1,266 @@ +use std::collections::{BTreeSet, HashSet}; + +use actix_web::{delete, get, post, put}; +use actix_web::{web, HttpResponse}; +use indexmap::IndexMap; +use meilisearch_core::{update, MainReader}; +use serde_json::Value; +use serde::Deserialize; + +use crate::Data; +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::routes::{IndexParam, IndexUpdateResponse}; + +type Document = IndexMap; + +#[derive(Deserialize)] +struct DocumentParam { + index_uid: String, + document_id: String, +} + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(get_document) + .service(delete_document) + .service(get_all_documents) + .service(add_documents) + .service(update_documents) + .service(delete_documents) + .service(clear_all_documents); +} + +#[get( + "/indexes/{index_uid}/documents/{document_id}", + wrap = "Authentication::Public" +)] +async fn get_document( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let reader = data.db.main_read_txn()?; + + let internal_id = index + .main + .external_to_internal_docid(&reader, &path.document_id)? + .ok_or(Error::document_not_found(&path.document_id))?; + + let document: Document = index + .document(&reader, None, internal_id)? + .ok_or(Error::document_not_found(&path.document_id))?; + + Ok(HttpResponse::Ok().json(document)) +} + +#[delete( + "/indexes/{index_uid}/documents/{document_id}", + wrap = "Authentication::Private" +)] +async fn delete_document( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let mut documents_deletion = index.documents_deletion(); + documents_deletion.delete_document_by_external_docid(path.document_id.clone()); + + let update_id = data.db.update_write(|w| documents_deletion.finalize(w))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct BrowseQuery { + offset: Option, + limit: Option, + attributes_to_retrieve: Option, +} + +pub fn get_all_documents_sync( + data: &web::Data, + reader: &MainReader, + index_uid: &str, + offset: usize, + limit: usize, + attributes_to_retrieve: Option<&String> +) -> Result, Error> { + let index = data + .db + .open_index(index_uid) + .ok_or(Error::index_not_found(index_uid))?; + + + let documents_ids: Result, _> = index + .documents_fields_counts + .documents_ids(reader)? + .skip(offset) + .take(limit) + .collect(); + + let attributes: Option> = attributes_to_retrieve + .map(|a| a.split(',').collect()); + + let mut documents = Vec::new(); + for document_id in documents_ids? { + if let Ok(Some(document)) = + index.document::(reader, attributes.as_ref(), document_id) + { + documents.push(document); + } + } + + Ok(documents) +} + +#[get("/indexes/{index_uid}/documents", wrap = "Authentication::Public")] +async fn get_all_documents( + data: web::Data, + path: web::Path, + params: web::Query, +) -> Result { + let offset = params.offset.unwrap_or(0); + let limit = params.limit.unwrap_or(20); + let index_uid = &path.index_uid; + let reader = data.db.main_read_txn()?; + + let documents = get_all_documents_sync( + &data, + &reader, + index_uid, + offset, + limit, + params.attributes_to_retrieve.as_ref() + )?; + + Ok(HttpResponse::Ok().json(documents)) +} + +fn find_primary_key(document: &IndexMap) -> Option { + for key in document.keys() { + if key.to_lowercase().contains("id") { + return Some(key.to_string()); + } + } + None +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct UpdateDocumentsQuery { + primary_key: Option, +} + +async fn update_multiple_documents( + data: web::Data, + path: web::Path, + params: web::Query, + body: web::Json>, + is_partial: bool, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + let reader = data.db.main_read_txn()?; + + let mut schema = index + .main + .schema(&reader)? + .ok_or(meilisearch_core::Error::SchemaMissing)?; + + if schema.primary_key().is_none() { + let id = match ¶ms.primary_key { + Some(id) => id.to_string(), + None => body + .first() + .and_then(find_primary_key) + .ok_or(meilisearch_core::Error::MissingPrimaryKey)?, + }; + + schema.set_primary_key(&id).map_err(Error::bad_request)?; + + data.db.main_write(|w| index.main.put_schema(w, &schema))?; + } + + let mut document_addition = if is_partial { + index.documents_partial_addition() + } else { + index.documents_addition() + }; + + for document in body.into_inner() { + document_addition.update_document(document); + } + + Ok(data.db.update_write(|w| document_addition.finalize(w))?) + })?; + return Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))); +} + +#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] +async fn add_documents( + data: web::Data, + path: web::Path, + params: web::Query, + body: web::Json>, +) -> Result { + update_multiple_documents(data, path, params, body, false).await +} + +#[put("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] +async fn update_documents( + data: web::Data, + path: web::Path, + params: web::Query, + body: web::Json>, +) -> Result { + update_multiple_documents(data, path, params, body, true).await +} + +#[post( + "/indexes/{index_uid}/documents/delete-batch", + wrap = "Authentication::Private" +)] +async fn delete_documents( + data: web::Data, + path: web::Path, + body: web::Json>, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let mut documents_deletion = index.documents_deletion(); + + for document_id in body.into_inner() { + let document_id = update::value_to_string(&document_id); + documents_deletion.delete_document_by_external_docid(document_id); + } + + let update_id = data.db.update_write(|w| documents_deletion.finalize(w))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] +async fn clear_all_documents( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let update_id = data.db.update_write(|w| index.clear_all(w))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} diff --git a/src/routes/dump.rs b/src/routes/dump.rs new file mode 100644 index 000000000..97fafdfa8 --- /dev/null +++ b/src/routes/dump.rs @@ -0,0 +1,64 @@ +use std::fs::File; +use std::path::Path; + +use actix_web::{get, post}; +use actix_web::{HttpResponse, web}; +use serde::{Deserialize, Serialize}; + +use crate::dump::{DumpInfo, DumpStatus, compressed_dumps_dir, init_dump_process}; +use crate::Data; +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(trigger_dump) + .service(get_dump_status); +} + +#[post("/dumps", wrap = "Authentication::Private")] +async fn trigger_dump( + data: web::Data, +) -> Result { + let dumps_dir = Path::new(&data.dumps_dir); + match init_dump_process(&data, &dumps_dir) { + Ok(resume) => Ok(HttpResponse::Accepted().json(resume)), + Err(e) => Err(e.into()) + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DumpStatusResponse { + status: String, +} + +#[derive(Deserialize)] +struct DumpParam { + dump_uid: String, +} + +#[get("/dumps/{dump_uid}/status", wrap = "Authentication::Private")] +async fn get_dump_status( + data: web::Data, + path: web::Path, +) -> Result { + let dumps_dir = Path::new(&data.dumps_dir); + let dump_uid = &path.dump_uid; + + if let Some(resume) = DumpInfo::get_current() { + if &resume.uid == dump_uid { + return Ok(HttpResponse::Ok().json(resume)); + } + } + + if File::open(compressed_dumps_dir(Path::new(dumps_dir), dump_uid)).is_ok() { + let resume = DumpInfo::new( + dump_uid.into(), + DumpStatus::Done + ); + + Ok(HttpResponse::Ok().json(resume)) + } else { + Err(Error::not_found("dump does not exist").into()) + } +} diff --git a/src/routes/health.rs b/src/routes/health.rs new file mode 100644 index 000000000..c57e4c7e9 --- /dev/null +++ b/src/routes/health.rs @@ -0,0 +1,13 @@ +use actix_web::get; +use actix_web::{web, HttpResponse}; + +use crate::error::ResponseError; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(get_health); +} + +#[get("/health")] +async fn get_health() -> Result { + Ok(HttpResponse::NoContent().finish()) +} diff --git a/src/routes/index.rs b/src/routes/index.rs new file mode 100644 index 000000000..aa0496920 --- /dev/null +++ b/src/routes/index.rs @@ -0,0 +1,388 @@ +use actix_web::{delete, get, post, put}; +use actix_web::{web, HttpResponse}; +use chrono::{DateTime, Utc}; +use log::error; +use meilisearch_core::{Database, MainReader, UpdateReader}; +use meilisearch_core::update::UpdateStatus; +use rand::seq::SliceRandom; +use serde::{Deserialize, Serialize}; + +use crate::Data; +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::routes::IndexParam; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(list_indexes) + .service(get_index) + .service(create_index) + .service(update_index) + .service(delete_index) + .service(get_update_status) + .service(get_all_updates_status); +} + +fn generate_uid() -> String { + let mut rng = rand::thread_rng(); + let sample = b"abcdefghijklmnopqrstuvwxyz0123456789"; + sample + .choose_multiple(&mut rng, 8) + .map(|c| *c as char) + .collect() +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IndexResponse { + pub name: String, + pub uid: String, + created_at: DateTime, + updated_at: DateTime, + pub primary_key: Option, +} + +pub fn list_indexes_sync(data: &web::Data, reader: &MainReader) -> Result, ResponseError> { + let mut indexes = Vec::new(); + + for index_uid in data.db.indexes_uids() { + let index = data.db.open_index(&index_uid); + + match index { + Some(index) => { + let name = index.main.name(reader)?.ok_or(Error::internal( + "Impossible to get the name of an index", + ))?; + let created_at = index + .main + .created_at(reader)? + .ok_or(Error::internal( + "Impossible to get the create date of an index", + ))?; + let updated_at = index + .main + .updated_at(reader)? + .ok_or(Error::internal( + "Impossible to get the last update date of an index", + ))?; + + let primary_key = match index.main.schema(reader) { + Ok(Some(schema)) => match schema.primary_key() { + Some(primary_key) => Some(primary_key.to_owned()), + None => None, + }, + _ => None, + }; + + let index_response = IndexResponse { + name, + uid: index_uid, + created_at, + updated_at, + primary_key, + }; + indexes.push(index_response); + } + None => error!( + "Index {} is referenced in the indexes list but cannot be found", + index_uid + ), + } + } + + Ok(indexes) +} + +#[get("/indexes", wrap = "Authentication::Private")] +async fn list_indexes(data: web::Data) -> Result { + let reader = data.db.main_read_txn()?; + let indexes = list_indexes_sync(&data, &reader)?; + + Ok(HttpResponse::Ok().json(indexes)) +} + +#[get("/indexes/{index_uid}", wrap = "Authentication::Private")] +async fn get_index( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let reader = data.db.main_read_txn()?; + let name = index.main.name(&reader)?.ok_or(Error::internal( + "Impossible to get the name of an index", + ))?; + let created_at = index + .main + .created_at(&reader)? + .ok_or(Error::internal( + "Impossible to get the create date of an index", + ))?; + let updated_at = index + .main + .updated_at(&reader)? + .ok_or(Error::internal( + "Impossible to get the last update date of an index", + ))?; + + let primary_key = match index.main.schema(&reader) { + Ok(Some(schema)) => match schema.primary_key() { + Some(primary_key) => Some(primary_key.to_owned()), + None => None, + }, + _ => None, + }; + let index_response = IndexResponse { + name, + uid: path.index_uid.clone(), + created_at, + updated_at, + primary_key, + }; + + Ok(HttpResponse::Ok().json(index_response)) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct IndexCreateRequest { + name: Option, + uid: Option, + primary_key: Option, +} + + +pub fn create_index_sync( + database: &std::sync::Arc, + uid: String, + name: String, + primary_key: Option, +) -> Result { + + let created_index = database + .create_index(&uid) + .map_err(|e| match e { + meilisearch_core::Error::IndexAlreadyExists => Error::IndexAlreadyExists(uid.clone()), + _ => Error::create_index(e) + })?; + + let index_response = database.main_write::<_, _, Error>(|mut write_txn| { + created_index.main.put_name(&mut write_txn, &name)?; + + let created_at = created_index + .main + .created_at(&write_txn)? + .ok_or(Error::internal("Impossible to read created at"))?; + + let updated_at = created_index + .main + .updated_at(&write_txn)? + .ok_or(Error::internal("Impossible to read updated at"))?; + + if let Some(id) = primary_key.clone() { + if let Some(mut schema) = created_index.main.schema(&write_txn)? { + schema + .set_primary_key(&id) + .map_err(Error::bad_request)?; + created_index.main.put_schema(&mut write_txn, &schema)?; + } + } + let index_response = IndexResponse { + name, + uid, + created_at, + updated_at, + primary_key, + }; + Ok(index_response) + })?; + + Ok(index_response) +} + +#[post("/indexes", wrap = "Authentication::Private")] +async fn create_index( + data: web::Data, + body: web::Json, +) -> Result { + if let (None, None) = (body.name.clone(), body.uid.clone()) { + return Err(Error::bad_request( + "Index creation must have an uid", + ).into()); + } + + let uid = match &body.uid { + Some(uid) => { + if uid + .chars() + .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') + { + uid.to_owned() + } else { + return Err(Error::InvalidIndexUid.into()); + } + } + None => loop { + let uid = generate_uid(); + if data.db.open_index(&uid).is_none() { + break uid; + } + }, + }; + + let name = body.name.as_ref().unwrap_or(&uid).to_string(); + + let index_response = create_index_sync(&data.db, uid, name, body.primary_key.clone())?; + + Ok(HttpResponse::Created().json(index_response)) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct UpdateIndexRequest { + name: Option, + primary_key: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct UpdateIndexResponse { + name: String, + uid: String, + created_at: DateTime, + updated_at: DateTime, + primary_key: Option, +} + +#[put("/indexes/{index_uid}", wrap = "Authentication::Private")] +async fn update_index( + data: web::Data, + path: web::Path, + body: web::Json, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + data.db.main_write::<_, _, ResponseError>(|writer| { + if let Some(name) = &body.name { + index.main.put_name(writer, name)?; + } + + if let Some(id) = body.primary_key.clone() { + if let Some(mut schema) = index.main.schema(writer)? { + schema.set_primary_key(&id)?; + index.main.put_schema(writer, &schema)?; + } + } + index.main.put_updated_at(writer)?; + Ok(()) + })?; + + let reader = data.db.main_read_txn()?; + let name = index.main.name(&reader)?.ok_or(Error::internal( + "Impossible to get the name of an index", + ))?; + let created_at = index + .main + .created_at(&reader)? + .ok_or(Error::internal( + "Impossible to get the create date of an index", + ))?; + let updated_at = index + .main + .updated_at(&reader)? + .ok_or(Error::internal( + "Impossible to get the last update date of an index", + ))?; + + let primary_key = match index.main.schema(&reader) { + Ok(Some(schema)) => match schema.primary_key() { + Some(primary_key) => Some(primary_key.to_owned()), + None => None, + }, + _ => None, + }; + + let index_response = IndexResponse { + name, + uid: path.index_uid.clone(), + created_at, + updated_at, + primary_key, + }; + + Ok(HttpResponse::Ok().json(index_response)) +} + +#[delete("/indexes/{index_uid}", wrap = "Authentication::Private")] +async fn delete_index( + data: web::Data, + path: web::Path, +) -> Result { + if data.db.delete_index(&path.index_uid)? { + Ok(HttpResponse::NoContent().finish()) + } else { + Err(Error::index_not_found(&path.index_uid).into()) + } +} + +#[derive(Deserialize)] +struct UpdateParam { + index_uid: String, + update_id: u64, +} + +#[get( + "/indexes/{index_uid}/updates/{update_id}", + wrap = "Authentication::Private" +)] +async fn get_update_status( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let reader = data.db.update_read_txn()?; + + let status = index.update_status(&reader, path.update_id)?; + + match status { + Some(status) => Ok(HttpResponse::Ok().json(status)), + None => Err(Error::NotFound(format!( + "Update {}", + path.update_id + )).into()), + } +} +pub fn get_all_updates_status_sync( + data: &web::Data, + reader: &UpdateReader, + index_uid: &str, +) -> Result, Error> { + let index = data + .db + .open_index(index_uid) + .ok_or(Error::index_not_found(index_uid))?; + + Ok(index.all_updates_status(reader)?) +} + +#[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] +async fn get_all_updates_status( + data: web::Data, + path: web::Path, +) -> Result { + + let reader = data.db.update_read_txn()?; + + let response = get_all_updates_status_sync(&data, &reader, &path.index_uid)?; + + Ok(HttpResponse::Ok().json(response)) +} diff --git a/src/routes/key.rs b/src/routes/key.rs new file mode 100644 index 000000000..a0cbaccc3 --- /dev/null +++ b/src/routes/key.rs @@ -0,0 +1,26 @@ +use actix_web::web; +use actix_web::HttpResponse; +use actix_web::get; +use serde::Serialize; + +use crate::helpers::Authentication; +use crate::Data; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(list); +} + +#[derive(Serialize)] +struct KeysResponse { + private: Option, + public: Option, +} + +#[get("/keys", wrap = "Authentication::Admin")] +async fn list(data: web::Data) -> HttpResponse { + let api_keys = data.api_keys.clone(); + HttpResponse::Ok().json(KeysResponse { + private: api_keys.private, + public: api_keys.public, + }) +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs new file mode 100644 index 000000000..15a858055 --- /dev/null +++ b/src/routes/mod.rs @@ -0,0 +1,44 @@ +use actix_web::{get, HttpResponse}; +use serde::{Deserialize, Serialize}; + +pub mod document; +pub mod health; +pub mod index; +pub mod key; +pub mod search; +pub mod setting; +pub mod stats; +pub mod stop_words; +pub mod synonym; +pub mod dump; + +#[derive(Deserialize)] +pub struct IndexParam { + index_uid: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct IndexUpdateResponse { + pub update_id: u64, +} + +impl IndexUpdateResponse { + pub fn with_id(update_id: u64) -> Self { + Self { update_id } + } +} + +#[get("/")] +pub async fn load_html() -> HttpResponse { + HttpResponse::Ok() + .content_type("text/html; charset=utf-8") + .body(include_str!("../../public/interface.html").to_string()) +} + +#[get("/bulma.min.css")] +pub async fn load_css() -> HttpResponse { + HttpResponse::Ok() + .content_type("text/css; charset=utf-8") + .body(include_str!("../../public/bulma.min.css").to_string()) +} diff --git a/src/routes/search.rs b/src/routes/search.rs new file mode 100644 index 000000000..3cd3c3f60 --- /dev/null +++ b/src/routes/search.rs @@ -0,0 +1,269 @@ +use std::collections::{HashMap, HashSet}; + +use actix_web::{get, post, web, HttpResponse}; +use log::warn; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::error::{Error, FacetCountError, ResponseError}; +use crate::helpers::meilisearch::{IndexSearchExt, SearchResult}; +use crate::helpers::Authentication; +use crate::routes::IndexParam; +use crate::Data; + +use meilisearch_core::facets::FacetFilter; +use meilisearch_schema::{FieldId, Schema}; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(search_with_post).service(search_with_url_query); +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchQuery { + q: Option, + offset: Option, + limit: Option, + attributes_to_retrieve: Option, + attributes_to_crop: Option, + crop_length: Option, + attributes_to_highlight: Option, + filters: Option, + matches: Option, + facet_filters: Option, + facets_distribution: Option, +} + +#[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")] +async fn search_with_url_query( + data: web::Data, + path: web::Path, + params: web::Query, +) -> Result { + let search_result = params.search(&path.index_uid, data)?; + Ok(HttpResponse::Ok().json(search_result)) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchQueryPost { + q: Option, + offset: Option, + limit: Option, + attributes_to_retrieve: Option>, + attributes_to_crop: Option>, + crop_length: Option, + attributes_to_highlight: Option>, + filters: Option, + matches: Option, + facet_filters: Option, + facets_distribution: Option>, +} + +impl From for SearchQuery { + fn from(other: SearchQueryPost) -> SearchQuery { + SearchQuery { + q: other.q, + offset: other.offset, + limit: other.limit, + attributes_to_retrieve: other.attributes_to_retrieve.map(|attrs| attrs.join(",")), + attributes_to_crop: other.attributes_to_crop.map(|attrs| attrs.join(",")), + crop_length: other.crop_length, + attributes_to_highlight: other.attributes_to_highlight.map(|attrs| attrs.join(",")), + filters: other.filters, + matches: other.matches, + facet_filters: other.facet_filters.map(|f| f.to_string()), + facets_distribution: other.facets_distribution.map(|f| format!("{:?}", f)), + } + } +} + +#[post("/indexes/{index_uid}/search", wrap = "Authentication::Public")] +async fn search_with_post( + data: web::Data, + path: web::Path, + params: web::Json, +) -> Result { + let query: SearchQuery = params.0.into(); + let search_result = query.search(&path.index_uid, data)?; + Ok(HttpResponse::Ok().json(search_result)) +} + +impl SearchQuery { + fn search( + &self, + index_uid: &str, + data: web::Data, + ) -> Result { + let index = data + .db + .open_index(index_uid) + .ok_or(Error::index_not_found(index_uid))?; + + let reader = data.db.main_read_txn()?; + let schema = index + .main + .schema(&reader)? + .ok_or(Error::internal("Impossible to retrieve the schema"))?; + + let query = self + .q + .clone() + .and_then(|q| if q.is_empty() { None } else { Some(q) }); + + let mut search_builder = index.new_search(query); + + if let Some(offset) = self.offset { + search_builder.offset(offset); + } + if let Some(limit) = self.limit { + search_builder.limit(limit); + } + + let available_attributes = schema.displayed_name(); + let mut restricted_attributes: HashSet<&str>; + match &self.attributes_to_retrieve { + Some(attributes_to_retrieve) => { + let attributes_to_retrieve: HashSet<&str> = + attributes_to_retrieve.split(',').collect(); + if attributes_to_retrieve.contains("*") { + restricted_attributes = available_attributes.clone(); + } else { + restricted_attributes = HashSet::new(); + for attr in attributes_to_retrieve { + if available_attributes.contains(attr) { + restricted_attributes.insert(attr); + search_builder.add_retrievable_field(attr.to_string()); + } else { + warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr); + } + } + } + } + None => { + restricted_attributes = available_attributes.clone(); + } + } + + if let Some(ref facet_filters) = self.facet_filters { + let attrs = index + .main + .attributes_for_faceting(&reader)? + .unwrap_or_default(); + search_builder.add_facet_filters(FacetFilter::from_str( + facet_filters, + &schema, + &attrs, + )?); + } + + if let Some(facets) = &self.facets_distribution { + match index.main.attributes_for_faceting(&reader)? { + Some(ref attrs) => { + let field_ids = prepare_facet_list(&facets, &schema, attrs)?; + search_builder.add_facets(field_ids); + } + None => return Err(FacetCountError::NoFacetSet.into()), + } + } + + if let Some(attributes_to_crop) = &self.attributes_to_crop { + let default_length = self.crop_length.unwrap_or(200); + let mut final_attributes: HashMap = HashMap::new(); + + for attribute in attributes_to_crop.split(',') { + let mut attribute = attribute.split(':'); + let attr = attribute.next(); + let length = attribute + .next() + .and_then(|s| s.parse().ok()) + .unwrap_or(default_length); + match attr { + Some("*") => { + for attr in &restricted_attributes { + final_attributes.insert(attr.to_string(), length); + } + } + Some(attr) => { + if available_attributes.contains(attr) { + final_attributes.insert(attr.to_string(), length); + } else { + warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr); + } + } + None => (), + } + } + search_builder.attributes_to_crop(final_attributes); + } + + if let Some(attributes_to_highlight) = &self.attributes_to_highlight { + let mut final_attributes: HashSet = HashSet::new(); + for attribute in attributes_to_highlight.split(',') { + if attribute == "*" { + for attr in &restricted_attributes { + final_attributes.insert(attr.to_string()); + } + } else if available_attributes.contains(attribute) { + final_attributes.insert(attribute.to_string()); + } else { + warn!("The attributes {:?} present in attributesToHighlight parameter doesn't exist", attribute); + } + } + + search_builder.attributes_to_highlight(final_attributes); + } + + if let Some(filters) = &self.filters { + search_builder.filters(filters.to_string()); + } + + if let Some(matches) = self.matches { + if matches { + search_builder.get_matches(); + } + } + search_builder.search(&reader) + } +} + +/// Parses the incoming string into an array of attributes for which to return a count. It returns +/// a Vec of attribute names ascociated with their id. +/// +/// An error is returned if the array is malformed, or if it contains attributes that are +/// unexisting, or not set as facets. +fn prepare_facet_list( + facets: &str, + schema: &Schema, + facet_attrs: &[FieldId], +) -> Result, FacetCountError> { + let json_array = serde_json::from_str(facets)?; + match json_array { + Value::Array(vals) => { + let wildcard = Value::String("*".to_string()); + if vals.iter().any(|f| f == &wildcard) { + let attrs = facet_attrs + .iter() + .filter_map(|&id| schema.name(id).map(|n| (id, n.to_string()))) + .collect(); + return Ok(attrs); + } + let mut field_ids = Vec::with_capacity(facet_attrs.len()); + for facet in vals { + match facet { + Value::String(facet) => { + if let Some(id) = schema.id(&facet) { + if !facet_attrs.contains(&id) { + return Err(FacetCountError::AttributeNotSet(facet)); + } + field_ids.push((id, facet)); + } + } + bad_val => return Err(FacetCountError::unexpected_token(bad_val, &["String"])), + } + } + Ok(field_ids) + } + bad_val => Err(FacetCountError::unexpected_token(bad_val, &["[String]"])), + } +} diff --git a/src/routes/setting.rs b/src/routes/setting.rs new file mode 100644 index 000000000..00562eed0 --- /dev/null +++ b/src/routes/setting.rs @@ -0,0 +1,547 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use actix_web::{delete, get, post}; +use actix_web::{web, HttpResponse}; +use meilisearch_core::{MainReader, UpdateWriter}; +use meilisearch_core::settings::{Settings, SettingsUpdate, UpdateState, DEFAULT_RANKING_RULES}; +use meilisearch_schema::Schema; + +use crate::Data; +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::routes::{IndexParam, IndexUpdateResponse}; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(update_all) + .service(get_all) + .service(delete_all) + .service(get_rules) + .service(update_rules) + .service(delete_rules) + .service(get_distinct) + .service(update_distinct) + .service(delete_distinct) + .service(get_searchable) + .service(update_searchable) + .service(delete_searchable) + .service(get_displayed) + .service(update_displayed) + .service(delete_displayed) + .service(get_attributes_for_faceting) + .service(delete_attributes_for_faceting) + .service(update_attributes_for_faceting); +} + +pub fn update_all_settings_txn( + data: &web::Data, + settings: SettingsUpdate, + index_uid: &str, + write_txn: &mut UpdateWriter, +) -> Result { + let index = data + .db + .open_index(index_uid) + .ok_or(Error::index_not_found(index_uid))?; + + let update_id = index.settings_update(write_txn, settings)?; + Ok(update_id) +} + +#[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] +async fn update_all( + data: web::Data, + path: web::Path, + body: web::Json, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + Ok(data.db.update_write::<_, _, ResponseError>(|writer| { + let settings = body.into_inner().to_update().map_err(Error::bad_request)?; + let update_id = index.settings_update(writer, settings)?; + Ok(update_id) + })?) + })?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +pub fn get_all_sync(data: &web::Data, reader: &MainReader, index_uid: &str) -> Result { + let index = data + .db + .open_index(index_uid) + .ok_or(Error::index_not_found(index_uid))?; + + let stop_words: BTreeSet = index.main.stop_words(&reader)?.into_iter().collect(); + + let synonyms_list = index.main.synonyms(reader)?; + + let mut synonyms = BTreeMap::new(); + let index_synonyms = &index.synonyms; + for synonym in synonyms_list { + let list = index_synonyms.synonyms(reader, synonym.as_bytes())?; + synonyms.insert(synonym, list); + } + + let ranking_rules = index + .main + .ranking_rules(reader)? + .unwrap_or(DEFAULT_RANKING_RULES.to_vec()) + .into_iter() + .map(|r| r.to_string()) + .collect(); + + let schema = index.main.schema(&reader)?; + + let distinct_attribute = match (index.main.distinct_attribute(reader)?, &schema) { + (Some(id), Some(schema)) => schema.name(id).map(str::to_string), + _ => None, + }; + + let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(&reader)?) { + (Some(schema), Some(attrs)) => attrs + .iter() + .filter_map(|&id| schema.name(id)) + .map(str::to_string) + .collect(), + _ => vec![], + }; + + let searchable_attributes = schema.as_ref().map(get_indexed_attributes); + let displayed_attributes = schema.as_ref().map(get_displayed_attributes); + + Ok(Settings { + ranking_rules: Some(Some(ranking_rules)), + distinct_attribute: Some(distinct_attribute), + searchable_attributes: Some(searchable_attributes), + displayed_attributes: Some(displayed_attributes), + stop_words: Some(Some(stop_words)), + synonyms: Some(Some(synonyms)), + attributes_for_faceting: Some(Some(attributes_for_faceting)), + }) +} + +#[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] +async fn get_all( + data: web::Data, + path: web::Path, +) -> Result { + let reader = data.db.main_read_txn()?; + let settings = get_all_sync(&data, &reader, &path.index_uid)?; + + Ok(HttpResponse::Ok().json(settings)) +} + +#[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] +async fn delete_all( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let settings = SettingsUpdate { + ranking_rules: UpdateState::Clear, + distinct_attribute: UpdateState::Clear, + primary_key: UpdateState::Clear, + searchable_attributes: UpdateState::Clear, + displayed_attributes: UpdateState::Clear, + stop_words: UpdateState::Clear, + synonyms: UpdateState::Clear, + attributes_for_faceting: UpdateState::Clear, + }; + + let update_id = data + .db + .update_write(|w| index.settings_update(w, settings))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[get( + "/indexes/{index_uid}/settings/ranking-rules", + wrap = "Authentication::Private" +)] +async fn get_rules( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + let reader = data.db.main_read_txn()?; + + let ranking_rules = index + .main + .ranking_rules(&reader)? + .unwrap_or(DEFAULT_RANKING_RULES.to_vec()) + .into_iter() + .map(|r| r.to_string()) + .collect::>(); + + Ok(HttpResponse::Ok().json(ranking_rules)) +} + +#[post( + "/indexes/{index_uid}/settings/ranking-rules", + wrap = "Authentication::Private" +)] +async fn update_rules( + data: web::Data, + path: web::Path, + body: web::Json>>, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + let settings = Settings { + ranking_rules: Some(body.into_inner()), + ..Settings::default() + }; + + let settings = settings.to_update().map_err(Error::bad_request)?; + Ok(data + .db + .update_write(|w| index.settings_update(w, settings))?) + })?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[delete( + "/indexes/{index_uid}/settings/ranking-rules", + wrap = "Authentication::Private" +)] +async fn delete_rules( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let settings = SettingsUpdate { + ranking_rules: UpdateState::Clear, + ..SettingsUpdate::default() + }; + + let update_id = data + .db + .update_write(|w| index.settings_update(w, settings))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[get( + "/indexes/{index_uid}/settings/distinct-attribute", + wrap = "Authentication::Private" +)] +async fn get_distinct( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + let reader = data.db.main_read_txn()?; + let distinct_attribute_id = index.main.distinct_attribute(&reader)?; + let schema = index.main.schema(&reader)?; + let distinct_attribute = match (schema, distinct_attribute_id) { + (Some(schema), Some(id)) => schema.name(id).map(str::to_string), + _ => None, + }; + + Ok(HttpResponse::Ok().json(distinct_attribute)) +} + +#[post( + "/indexes/{index_uid}/settings/distinct-attribute", + wrap = "Authentication::Private" +)] +async fn update_distinct( + data: web::Data, + path: web::Path, + body: web::Json>, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + let settings = Settings { + distinct_attribute: Some(body.into_inner()), + ..Settings::default() + }; + + let settings = settings.to_update().map_err(Error::bad_request)?; + Ok(data + .db + .update_write(|w| index.settings_update(w, settings))?) + })?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[delete( + "/indexes/{index_uid}/settings/distinct-attribute", + wrap = "Authentication::Private" +)] +async fn delete_distinct( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let settings = SettingsUpdate { + distinct_attribute: UpdateState::Clear, + ..SettingsUpdate::default() + }; + + let update_id = data + .db + .update_write(|w| index.settings_update(w, settings))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[get( + "/indexes/{index_uid}/settings/searchable-attributes", + wrap = "Authentication::Private" +)] +async fn get_searchable( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + let reader = data.db.main_read_txn()?; + let schema = index.main.schema(&reader)?; + let searchable_attributes: Option> = schema.as_ref().map(get_indexed_attributes); + + Ok(HttpResponse::Ok().json(searchable_attributes)) +} + +#[post( + "/indexes/{index_uid}/settings/searchable-attributes", + wrap = "Authentication::Private" +)] +async fn update_searchable( + data: web::Data, + path: web::Path, + body: web::Json>>, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + let settings = Settings { + searchable_attributes: Some(body.into_inner()), + ..Settings::default() + }; + + let settings = settings.to_update().map_err(Error::bad_request)?; + + Ok(data + .db + .update_write(|w| index.settings_update(w, settings))?) + })?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[delete( + "/indexes/{index_uid}/settings/searchable-attributes", + wrap = "Authentication::Private" +)] +async fn delete_searchable( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let settings = SettingsUpdate { + searchable_attributes: UpdateState::Clear, + ..SettingsUpdate::default() + }; + + let update_id = data + .db + .update_write(|w| index.settings_update(w, settings))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[get( + "/indexes/{index_uid}/settings/displayed-attributes", + wrap = "Authentication::Private" +)] +async fn get_displayed( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + let reader = data.db.main_read_txn()?; + + let schema = index.main.schema(&reader)?; + + let displayed_attributes = schema.as_ref().map(get_displayed_attributes); + + Ok(HttpResponse::Ok().json(displayed_attributes)) +} + +#[post( + "/indexes/{index_uid}/settings/displayed-attributes", + wrap = "Authentication::Private" +)] +async fn update_displayed( + data: web::Data, + path: web::Path, + body: web::Json>>, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + let settings = Settings { + displayed_attributes: Some(body.into_inner()), + ..Settings::default() + }; + + let settings = settings.to_update().map_err(Error::bad_request)?; + Ok(data + .db + .update_write(|w| index.settings_update(w, settings))?) + })?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[delete( + "/indexes/{index_uid}/settings/displayed-attributes", + wrap = "Authentication::Private" +)] +async fn delete_displayed( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let settings = SettingsUpdate { + displayed_attributes: UpdateState::Clear, + ..SettingsUpdate::default() + }; + + let update_id = data + .db + .update_write(|w| index.settings_update(w, settings))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[get( + "/indexes/{index_uid}/settings/attributes-for-faceting", + wrap = "Authentication::Private" +)] +async fn get_attributes_for_faceting( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let attributes_for_faceting = data.db.main_read::<_, _, ResponseError>(|reader| { + let schema = index.main.schema(reader)?; + let attrs = index.main.attributes_for_faceting(reader)?; + let attr_names = match (&schema, &attrs) { + (Some(schema), Some(attrs)) => attrs + .iter() + .filter_map(|&id| schema.name(id)) + .map(str::to_string) + .collect(), + _ => vec![], + }; + Ok(attr_names) + })?; + + Ok(HttpResponse::Ok().json(attributes_for_faceting)) +} + +#[post( + "/indexes/{index_uid}/settings/attributes-for-faceting", + wrap = "Authentication::Private" +)] +async fn update_attributes_for_faceting( + data: web::Data, + path: web::Path, + body: web::Json>>, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + let settings = Settings { + attributes_for_faceting: Some(body.into_inner()), + ..Settings::default() + }; + + let settings = settings.to_update().map_err(Error::bad_request)?; + Ok(data + .db + .update_write(|w| index.settings_update(w, settings))?) + })?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[delete( + "/indexes/{index_uid}/settings/attributes-for-faceting", + wrap = "Authentication::Private" +)] +async fn delete_attributes_for_faceting( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let settings = SettingsUpdate { + attributes_for_faceting: UpdateState::Clear, + ..SettingsUpdate::default() + }; + + let update_id = data + .db + .update_write(|w| index.settings_update(w, settings))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +fn get_indexed_attributes(schema: &Schema) -> Vec { + if schema.is_indexed_all() { + ["*"].iter().map(|s| s.to_string()).collect() + } else { + schema + .indexed_name() + .iter() + .map(|s| s.to_string()) + .collect() + } +} + +fn get_displayed_attributes(schema: &Schema) -> BTreeSet { + if schema.is_displayed_all() { + ["*"].iter().map(|s| s.to_string()).collect() + } else { + schema + .displayed_name() + .iter() + .map(|s| s.to_string()) + .collect() + } +} diff --git a/src/routes/stats.rs b/src/routes/stats.rs new file mode 100644 index 000000000..f8c531732 --- /dev/null +++ b/src/routes/stats.rs @@ -0,0 +1,134 @@ +use std::collections::{HashMap, BTreeMap}; + +use actix_web::web; +use actix_web::HttpResponse; +use actix_web::get; +use chrono::{DateTime, Utc}; +use log::error; +use serde::Serialize; +use walkdir::WalkDir; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::routes::IndexParam; +use crate::Data; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(index_stats) + .service(get_stats) + .service(get_version); +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct IndexStatsResponse { + number_of_documents: u64, + is_indexing: bool, + fields_distribution: BTreeMap, +} + +#[get("/indexes/{index_uid}/stats", wrap = "Authentication::Private")] +async fn index_stats( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let reader = data.db.main_read_txn()?; + + let number_of_documents = index.main.number_of_documents(&reader)?; + + let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default(); + + let update_reader = data.db.update_read_txn()?; + + let is_indexing = + data.db.is_indexing(&update_reader, &path.index_uid)? + .ok_or(Error::internal( + "Impossible to know if the database is indexing", + ))?; + + Ok(HttpResponse::Ok().json(IndexStatsResponse { + number_of_documents, + is_indexing, + fields_distribution, + })) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct StatsResult { + database_size: u64, + last_update: Option>, + indexes: HashMap, +} + +#[get("/stats", wrap = "Authentication::Private")] +async fn get_stats(data: web::Data) -> Result { + let mut index_list = HashMap::new(); + + let reader = data.db.main_read_txn()?; + let update_reader = data.db.update_read_txn()?; + + let indexes_set = data.db.indexes_uids(); + for index_uid in indexes_set { + let index = data.db.open_index(&index_uid); + match index { + Some(index) => { + let number_of_documents = index.main.number_of_documents(&reader)?; + + let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default(); + + let is_indexing = data.db.is_indexing(&update_reader, &index_uid)?.ok_or( + Error::internal("Impossible to know if the database is indexing"), + )?; + + let response = IndexStatsResponse { + number_of_documents, + is_indexing, + fields_distribution, + }; + index_list.insert(index_uid, response); + } + None => error!( + "Index {:?} is referenced in the indexes list but cannot be found", + index_uid + ), + } + } + + let database_size = WalkDir::new(&data.db_path) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len()); + + let last_update = data.db.last_update(&reader)?; + + Ok(HttpResponse::Ok().json(StatsResult { + database_size, + last_update, + indexes: index_list, + })) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct VersionResponse { + commit_sha: String, + build_date: String, + pkg_version: String, +} + +#[get("/version", wrap = "Authentication::Private")] +async fn get_version() -> HttpResponse { + HttpResponse::Ok().json(VersionResponse { + commit_sha: env!("VERGEN_SHA").to_string(), + build_date: env!("VERGEN_BUILD_TIMESTAMP").to_string(), + pkg_version: env!("CARGO_PKG_VERSION").to_string(), + }) +} diff --git a/src/routes/stop_words.rs b/src/routes/stop_words.rs new file mode 100644 index 000000000..c757b4d14 --- /dev/null +++ b/src/routes/stop_words.rs @@ -0,0 +1,79 @@ +use actix_web::{web, HttpResponse}; +use actix_web::{delete, get, post}; +use meilisearch_core::settings::{SettingsUpdate, UpdateState}; +use std::collections::BTreeSet; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::routes::{IndexParam, IndexUpdateResponse}; +use crate::Data; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(get).service(update).service(delete); +} + +#[get( + "/indexes/{index_uid}/settings/stop-words", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + let reader = data.db.main_read_txn()?; + let stop_words = index.main.stop_words(&reader)?; + + Ok(HttpResponse::Ok().json(stop_words)) +} + +#[post( + "/indexes/{index_uid}/settings/stop-words", + wrap = "Authentication::Private" +)] +async fn update( + data: web::Data, + path: web::Path, + body: web::Json>, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + let settings = SettingsUpdate { + stop_words: UpdateState::Update(body.into_inner()), + ..SettingsUpdate::default() + }; + + Ok(data + .db + .update_write(|w| index.settings_update(w, settings))?) + })?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[delete( + "/indexes/{index_uid}/settings/stop-words", + wrap = "Authentication::Private" +)] +async fn delete( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let settings = SettingsUpdate { + stop_words: UpdateState::Clear, + ..SettingsUpdate::default() + }; + + let update_id = data + .db + .update_write(|w| index.settings_update(w, settings))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} diff --git a/src/routes/synonym.rs b/src/routes/synonym.rs new file mode 100644 index 000000000..5aefaaca5 --- /dev/null +++ b/src/routes/synonym.rs @@ -0,0 +1,90 @@ +use std::collections::BTreeMap; + +use actix_web::{web, HttpResponse}; +use actix_web::{delete, get, post}; +use indexmap::IndexMap; +use meilisearch_core::settings::{SettingsUpdate, UpdateState}; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::routes::{IndexParam, IndexUpdateResponse}; +use crate::Data; + +pub fn services(cfg: &mut web::ServiceConfig) { + cfg.service(get).service(update).service(delete); +} + +#[get( + "/indexes/{index_uid}/settings/synonyms", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let reader = data.db.main_read_txn()?; + + let synonyms_list = index.main.synonyms(&reader)?; + + let mut synonyms = IndexMap::new(); + let index_synonyms = &index.synonyms; + for synonym in synonyms_list { + let list = index_synonyms.synonyms(&reader, synonym.as_bytes())?; + synonyms.insert(synonym, list); + } + + Ok(HttpResponse::Ok().json(synonyms)) +} + +#[post( + "/indexes/{index_uid}/settings/synonyms", + wrap = "Authentication::Private" +)] +async fn update( + data: web::Data, + path: web::Path, + body: web::Json>>, +) -> Result { + let update_id = data.get_or_create_index(&path.index_uid, |index| { + let settings = SettingsUpdate { + synonyms: UpdateState::Update(body.into_inner()), + ..SettingsUpdate::default() + }; + + Ok(data + .db + .update_write(|w| index.settings_update(w, settings))?) + })?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} + +#[delete( + "/indexes/{index_uid}/settings/synonyms", + wrap = "Authentication::Private" +)] +async fn delete( + data: web::Data, + path: web::Path, +) -> Result { + let index = data + .db + .open_index(&path.index_uid) + .ok_or(Error::index_not_found(&path.index_uid))?; + + let settings = SettingsUpdate { + synonyms: UpdateState::Clear, + ..SettingsUpdate::default() + }; + + let update_id = data + .db + .update_write(|w| index.settings_update(w, settings))?; + + Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) +} diff --git a/src/snapshot.rs b/src/snapshot.rs new file mode 100644 index 000000000..520044f84 --- /dev/null +++ b/src/snapshot.rs @@ -0,0 +1,96 @@ +use crate::Data; +use crate::error::Error; +use crate::helpers::compression; + +use log::error; +use std::fs::create_dir_all; +use std::path::Path; +use std::thread; +use std::time::{Duration}; +use tempfile::TempDir; + +pub fn load_snapshot( + db_path: &str, + snapshot_path: &Path, + ignore_snapshot_if_db_exists: bool, + ignore_missing_snapshot: bool +) -> Result<(), Error> { + let db_path = Path::new(db_path); + + if !db_path.exists() && snapshot_path.exists() { + compression::from_tar_gz(snapshot_path, db_path) + } else if db_path.exists() && !ignore_snapshot_if_db_exists { + Err(Error::Internal(format!("database already exists at {:?}, try to delete it or rename it", db_path.canonicalize().unwrap_or(db_path.into())))) + } else if !snapshot_path.exists() && !ignore_missing_snapshot { + Err(Error::Internal(format!("snapshot doesn't exist at {:?}", snapshot_path.canonicalize().unwrap_or(snapshot_path.into())))) + } else { + Ok(()) + } +} + +pub fn create_snapshot(data: &Data, snapshot_path: &Path) -> Result<(), Error> { + let tmp_dir = TempDir::new()?; + + data.db.copy_and_compact_to_path(tmp_dir.path())?; + + compression::to_tar_gz(tmp_dir.path(), snapshot_path).map_err(|e| Error::Internal(format!("something went wrong during snapshot compression: {}", e))) +} + +pub fn schedule_snapshot(data: Data, snapshot_dir: &Path, time_gap_s: u64) -> Result<(), Error> { + if snapshot_dir.file_name().is_none() { + return Err(Error::Internal("invalid snapshot file path".to_string())); + } + let db_name = Path::new(&data.db_path).file_name().ok_or_else(|| Error::Internal("invalid database name".to_string()))?; + create_dir_all(snapshot_dir)?; + let snapshot_path = snapshot_dir.join(format!("{}.snapshot", db_name.to_str().unwrap_or("data.ms"))); + + thread::spawn(move || loop { + if let Err(e) = create_snapshot(&data, &snapshot_path) { + error!("Unsuccessful snapshot creation: {}", e); + } + thread::sleep(Duration::from_secs(time_gap_s)); + }); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::prelude::*; + use std::fs; + + #[test] + fn test_pack_unpack() { + let tempdir = TempDir::new().unwrap(); + + let test_dir = tempdir.path(); + let src_dir = test_dir.join("src"); + let dest_dir = test_dir.join("complex/destination/path/"); + let archive_path = test_dir.join("archive.snapshot"); + + let file_1_relative = Path::new("file1.txt"); + let subdir_relative = Path::new("subdir/"); + let file_2_relative = Path::new("subdir/file2.txt"); + + create_dir_all(src_dir.join(subdir_relative)).unwrap(); + fs::File::create(src_dir.join(file_1_relative)).unwrap().write_all(b"Hello_file_1").unwrap(); + fs::File::create(src_dir.join(file_2_relative)).unwrap().write_all(b"Hello_file_2").unwrap(); + + + assert!(compression::to_tar_gz(&src_dir, &archive_path).is_ok()); + assert!(archive_path.exists()); + assert!(load_snapshot(&dest_dir.to_str().unwrap(), &archive_path, false, false).is_ok()); + + assert!(dest_dir.exists()); + assert!(dest_dir.join(file_1_relative).exists()); + assert!(dest_dir.join(subdir_relative).exists()); + assert!(dest_dir.join(file_2_relative).exists()); + + let contents = fs::read_to_string(dest_dir.join(file_1_relative)).unwrap(); + assert_eq!(contents, "Hello_file_1"); + + let contents = fs::read_to_string(dest_dir.join(file_2_relative)).unwrap(); + assert_eq!(contents, "Hello_file_2"); + } +} diff --git a/tests/assets/dumps/v1/metadata.json b/tests/assets/dumps/v1/metadata.json new file mode 100644 index 000000000..6fe302324 --- /dev/null +++ b/tests/assets/dumps/v1/metadata.json @@ -0,0 +1,12 @@ +{ + "indices": [{ + "uid": "test", + "primaryKey": "id" + }, { + "uid": "test2", + "primaryKey": "test2_id" + } + ], + "dbVersion": "0.13.0", + "dumpVersion": "1" +} diff --git a/tests/assets/dumps/v1/test/documents.jsonl b/tests/assets/dumps/v1/test/documents.jsonl new file mode 100644 index 000000000..7af80f342 --- /dev/null +++ b/tests/assets/dumps/v1/test/documents.jsonl @@ -0,0 +1,77 @@ +{"id":0,"isActive":false,"balance":"$2,668.55","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Lucas Hess","gender":"male","email":"lucashess@chorizon.com","phone":"+1 (998) 478-2597","address":"412 Losee Terrace, Blairstown, Georgia, 2825","about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n","registered":"2016-06-21T09:30:25 -02:00","latitude":-44.174957,"longitude":-145.725388,"tags":["bug","bug"]} +{"id":1,"isActive":true,"balance":"$1,706.13","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Cherry Orr","gender":"female","email":"cherryorr@chorizon.com","phone":"+1 (995) 479-3174","address":"442 Beverly Road, Ventress, New Mexico, 3361","about":"Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n","registered":"2020-03-18T11:12:21 -01:00","latitude":-24.356932,"longitude":27.184808,"tags":["new issue","bug"]} +{"id":2,"isActive":true,"balance":"$2,467.47","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Patricia Goff","gender":"female","email":"patriciagoff@chorizon.com","phone":"+1 (864) 463-2277","address":"866 Hornell Loop, Cresaptown, Ohio, 1700","about":"Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n","registered":"2014-10-28T12:59:30 -01:00","latitude":-64.008555,"longitude":11.867098,"tags":["good first issue"]} +{"id":3,"isActive":true,"balance":"$3,344.40","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Adeline Flynn","gender":"female","email":"adelineflynn@chorizon.com","phone":"+1 (994) 600-2840","address":"428 Paerdegat Avenue, Hollymead, Pennsylvania, 948","about":"Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n","registered":"2014-03-27T06:24:45 -01:00","latitude":-74.485173,"longitude":-11.059859,"tags":["bug","good first issue","wontfix","new issue"]} +{"id":4,"isActive":false,"balance":"$2,575.78","picture":"http://placehold.it/32x32","age":39,"color":"Green","name":"Mariana Pacheco","gender":"female","email":"marianapacheco@chorizon.com","phone":"+1 (820) 414-2223","address":"664 Rapelye Street, Faywood, California, 7320","about":"Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n","registered":"2015-09-02T03:23:35 -02:00","latitude":75.763501,"longitude":-78.777124,"tags":["new issue"]} +{"id":5,"isActive":true,"balance":"$3,793.09","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Warren Watson","gender":"male","email":"warrenwatson@chorizon.com","phone":"+1 (807) 583-2427","address":"671 Prince Street, Faxon, Connecticut, 4275","about":"Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n","registered":"2017-06-04T06:02:17 -02:00","latitude":29.979223,"longitude":25.358943,"tags":["wontfix","wontfix","wontfix"]} +{"id":6,"isActive":true,"balance":"$2,919.70","picture":"http://placehold.it/32x32","age":20,"color":"blue","name":"Shelia Berry","gender":"female","email":"sheliaberry@chorizon.com","phone":"+1 (853) 511-2651","address":"437 Forrest Street, Coventry, Illinois, 2056","about":"Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n","registered":"2018-07-11T02:45:01 -02:00","latitude":54.815991,"longitude":-118.690609,"tags":["good first issue","bug","wontfix","new issue"]} +{"id":7,"isActive":true,"balance":"$1,349.50","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Chrystal Boyd","gender":"female","email":"chrystalboyd@chorizon.com","phone":"+1 (936) 563-2802","address":"670 Croton Loop, Sussex, Florida, 4692","about":"Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n","registered":"2016-11-01T07:36:04 -01:00","latitude":-24.711933,"longitude":147.246705,"tags":[]} +{"id":8,"isActive":false,"balance":"$3,999.56","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Martin Porter","gender":"male","email":"martinporter@chorizon.com","phone":"+1 (895) 580-2304","address":"577 Regent Place, Aguila, Guam, 6554","about":"Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n","registered":"2014-09-20T02:08:30 -02:00","latitude":-88.344273,"longitude":37.964466,"tags":[]} +{"id":9,"isActive":true,"balance":"$3,729.71","picture":"http://placehold.it/32x32","age":26,"color":"blue","name":"Kelli Mendez","gender":"female","email":"kellimendez@chorizon.com","phone":"+1 (936) 401-2236","address":"242 Caton Place, Grazierville, Alabama, 3968","about":"Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n","registered":"2018-05-04T10:35:30 -02:00","latitude":49.37551,"longitude":41.872323,"tags":["new issue","new issue"]} +{"id":10,"isActive":false,"balance":"$1,127.47","picture":"http://placehold.it/32x32","age":27,"color":"blue","name":"Maddox Johns","gender":"male","email":"maddoxjohns@chorizon.com","phone":"+1 (892) 470-2357","address":"756 Beard Street, Avalon, Louisiana, 114","about":"Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n","registered":"2016-04-22T06:41:25 -02:00","latitude":66.640229,"longitude":-17.222666,"tags":["new issue","good first issue","good first issue","new issue"]} +{"id":11,"isActive":true,"balance":"$1,351.43","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Evans Wagner","gender":"male","email":"evanswagner@chorizon.com","phone":"+1 (889) 496-2332","address":"118 Monaco Place, Lutsen, Delaware, 6209","about":"Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n","registered":"2016-10-27T01:26:31 -02:00","latitude":-77.673222,"longitude":-142.657214,"tags":["good first issue","good first issue"]} +{"id":12,"isActive":false,"balance":"$3,394.96","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Aida Kirby","gender":"female","email":"aidakirby@chorizon.com","phone":"+1 (942) 532-2325","address":"797 Engert Avenue, Wilsonia, Idaho, 6532","about":"Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n","registered":"2018-06-18T04:39:57 -02:00","latitude":-58.062041,"longitude":34.999254,"tags":["new issue","wontfix","bug","new issue"]} +{"id":13,"isActive":true,"balance":"$2,812.62","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Nelda Burris","gender":"female","email":"neldaburris@chorizon.com","phone":"+1 (813) 600-2576","address":"160 Opal Court, Fowlerville, Tennessee, 2170","about":"Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n","registered":"2015-08-15T12:39:53 -02:00","latitude":66.6871,"longitude":179.549488,"tags":["wontfix"]} +{"id":14,"isActive":true,"balance":"$1,718.33","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Jennifer Hart","gender":"female","email":"jenniferhart@chorizon.com","phone":"+1 (850) 537-2513","address":"124 Veranda Place, Nash, Utah, 985","about":"Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n","registered":"2016-09-04T11:46:59 -02:00","latitude":-66.827751,"longitude":99.220079,"tags":["wontfix","bug","new issue","new issue"]} +{"id":15,"isActive":false,"balance":"$2,698.16","picture":"http://placehold.it/32x32","age":28,"color":"blue","name":"Aurelia Contreras","gender":"female","email":"aureliacontreras@chorizon.com","phone":"+1 (932) 442-3103","address":"655 Dwight Street, Grapeview, Palau, 8356","about":"Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n","registered":"2014-09-11T10:43:15 -02:00","latitude":-71.328973,"longitude":133.404895,"tags":["wontfix","bug","good first issue"]} +{"id":16,"isActive":true,"balance":"$3,303.25","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Estella Bass","gender":"female","email":"estellabass@chorizon.com","phone":"+1 (825) 436-2909","address":"435 Rockwell Place, Garberville, Wisconsin, 2230","about":"Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n","registered":"2017-11-23T09:32:09 -01:00","latitude":81.17014,"longitude":-145.262693,"tags":["new issue"]} +{"id":17,"isActive":false,"balance":"$3,579.20","picture":"http://placehold.it/32x32","age":25,"color":"brown","name":"Ortega Brennan","gender":"male","email":"ortegabrennan@chorizon.com","phone":"+1 (906) 526-2287","address":"440 Berry Street, Rivera, Maine, 1849","about":"Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n","registered":"2016-03-31T02:17:13 -02:00","latitude":-68.407524,"longitude":-113.642067,"tags":["new issue","wontfix"]} +{"id":18,"isActive":false,"balance":"$1,484.92","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Leonard Tillman","gender":"male","email":"leonardtillman@chorizon.com","phone":"+1 (864) 541-3456","address":"985 Provost Street, Charco, New Hampshire, 8632","about":"Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n","registered":"2018-05-06T08:21:27 -02:00","latitude":-8.581801,"longitude":-61.910062,"tags":["wontfix","new issue","bug","bug"]} +{"id":19,"isActive":true,"balance":"$3,572.55","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Dale Payne","gender":"male","email":"dalepayne@chorizon.com","phone":"+1 (814) 469-3499","address":"536 Dare Court, Ironton, Arkansas, 8605","about":"Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n","registered":"2019-10-11T01:01:33 -02:00","latitude":-18.280968,"longitude":-126.091797,"tags":["bug","wontfix","wontfix","wontfix"]} +{"id":20,"isActive":true,"balance":"$1,986.48","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Florence Long","gender":"female","email":"florencelong@chorizon.com","phone":"+1 (972) 557-3858","address":"519 Hendrickson Street, Templeton, Hawaii, 2389","about":"Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n","registered":"2016-05-02T09:18:59 -02:00","latitude":-27.110866,"longitude":-45.09445,"tags":[]} +{"id":21,"isActive":true,"balance":"$1,440.09","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Levy Whitley","gender":"male","email":"levywhitley@chorizon.com","phone":"+1 (911) 458-2411","address":"187 Thomas Street, Hachita, North Carolina, 2989","about":"Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n","registered":"2014-04-30T07:31:38 -02:00","latitude":-6.537315,"longitude":171.813536,"tags":["bug"]} +{"id":22,"isActive":false,"balance":"$2,938.57","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Bernard Mcfarland","gender":"male","email":"bernardmcfarland@chorizon.com","phone":"+1 (979) 442-3386","address":"409 Hall Street, Keyport, Federated States Of Micronesia, 7011","about":"Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n","registered":"2017-08-10T10:07:59 -02:00","latitude":63.766795,"longitude":68.177069,"tags":[]} +{"id":23,"isActive":true,"balance":"$1,678.49","picture":"http://placehold.it/32x32","age":31,"color":"brown","name":"Blanca Mcclain","gender":"female","email":"blancamcclain@chorizon.com","phone":"+1 (976) 439-2772","address":"176 Crooke Avenue, Valle, Virginia, 5373","about":"Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n","registered":"2015-10-12T11:57:28 -02:00","latitude":-8.944564,"longitude":-150.711709,"tags":["bug","wontfix","good first issue"]} +{"id":24,"isActive":true,"balance":"$2,276.87","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Espinoza Ford","gender":"male","email":"espinozaford@chorizon.com","phone":"+1 (945) 429-3975","address":"137 Bowery Street, Itmann, District Of Columbia, 1864","about":"Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n","registered":"2014-03-26T02:16:08 -01:00","latitude":-37.137666,"longitude":-51.811757,"tags":["wontfix","bug"]} +{"id":25,"isActive":true,"balance":"$3,973.43","picture":"http://placehold.it/32x32","age":29,"color":"Green","name":"Sykes Conley","gender":"male","email":"sykesconley@chorizon.com","phone":"+1 (851) 401-3916","address":"345 Grand Street, Woodlands, Missouri, 4461","about":"Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n","registered":"2015-09-12T06:03:56 -02:00","latitude":67.282955,"longitude":-64.341323,"tags":["wontfix"]} +{"id":26,"isActive":false,"balance":"$1,431.50","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Barlow Duran","gender":"male","email":"barlowduran@chorizon.com","phone":"+1 (995) 436-2562","address":"481 Everett Avenue, Allison, Nebraska, 3065","about":"Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n","registered":"2017-06-29T04:28:43 -02:00","latitude":-38.70606,"longitude":55.02816,"tags":["new issue"]} +{"id":27,"isActive":true,"balance":"$3,478.27","picture":"http://placehold.it/32x32","age":31,"color":"blue","name":"Schwartz Morgan","gender":"male","email":"schwartzmorgan@chorizon.com","phone":"+1 (861) 507-2067","address":"451 Lincoln Road, Fairlee, Washington, 2717","about":"Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n","registered":"2016-05-10T08:34:54 -02:00","latitude":-75.886403,"longitude":93.044471,"tags":["bug","bug","wontfix","wontfix"]} +{"id":28,"isActive":true,"balance":"$2,825.59","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Kristy Leon","gender":"female","email":"kristyleon@chorizon.com","phone":"+1 (948) 465-2563","address":"594 Macon Street, Floris, South Dakota, 3565","about":"Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n","registered":"2014-12-14T04:10:29 -01:00","latitude":-50.01615,"longitude":-68.908804,"tags":["wontfix","good first issue"]} +{"id":29,"isActive":false,"balance":"$3,028.03","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Ashley Pittman","gender":"male","email":"ashleypittman@chorizon.com","phone":"+1 (928) 507-3523","address":"646 Adelphi Street, Clara, Colorado, 6056","about":"Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n","registered":"2016-01-07T10:40:48 -01:00","latitude":-58.766037,"longitude":-124.828485,"tags":["wontfix"]} +{"id":30,"isActive":true,"balance":"$2,021.11","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Stacy Espinoza","gender":"female","email":"stacyespinoza@chorizon.com","phone":"+1 (999) 487-3253","address":"931 Alabama Avenue, Bangor, Alaska, 8215","about":"Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n","registered":"2014-07-16T06:15:53 -02:00","latitude":41.560197,"longitude":177.697,"tags":["new issue","new issue","bug"]} +{"id":31,"isActive":false,"balance":"$3,609.82","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Vilma Garza","gender":"female","email":"vilmagarza@chorizon.com","phone":"+1 (944) 585-2021","address":"565 Tech Place, Sedley, Puerto Rico, 858","about":"Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n","registered":"2017-06-30T07:43:52 -02:00","latitude":-12.574889,"longitude":-54.771186,"tags":["new issue","wontfix","wontfix"]} +{"id":32,"isActive":false,"balance":"$2,882.34","picture":"http://placehold.it/32x32","age":38,"color":"brown","name":"June Dunlap","gender":"female","email":"junedunlap@chorizon.com","phone":"+1 (997) 504-2937","address":"353 Cozine Avenue, Goodville, Indiana, 1438","about":"Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n","registered":"2016-08-23T08:54:11 -02:00","latitude":-27.883363,"longitude":-163.919683,"tags":["new issue","new issue","bug","wontfix"]} +{"id":33,"isActive":true,"balance":"$3,556.54","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Cecilia Greer","gender":"female","email":"ceciliagreer@chorizon.com","phone":"+1 (977) 573-3498","address":"696 Withers Street, Lydia, Oklahoma, 3220","about":"Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n","registered":"2017-01-13T11:30:12 -01:00","latitude":60.467215,"longitude":84.684575,"tags":["wontfix","good first issue","good first issue","wontfix"]} +{"id":34,"isActive":true,"balance":"$1,413.35","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Mckay Schroeder","gender":"male","email":"mckayschroeder@chorizon.com","phone":"+1 (816) 480-3657","address":"958 Miami Court, Rehrersburg, Northern Mariana Islands, 567","about":"Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n","registered":"2016-02-08T04:50:15 -01:00","latitude":-72.413287,"longitude":-159.254371,"tags":["good first issue"]} +{"id":35,"isActive":true,"balance":"$2,306.53","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Sawyer Mccormick","gender":"male","email":"sawyermccormick@chorizon.com","phone":"+1 (829) 569-3012","address":"749 Apollo Street, Eastvale, Texas, 7373","about":"Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n","registered":"2019-11-30T11:53:23 -01:00","latitude":-48.978194,"longitude":110.950191,"tags":["good first issue","new issue","new issue","bug"]} +{"id":36,"isActive":false,"balance":"$1,844.54","picture":"http://placehold.it/32x32","age":37,"color":"brown","name":"Barbra Valenzuela","gender":"female","email":"barbravalenzuela@chorizon.com","phone":"+1 (992) 512-2649","address":"617 Schenck Court, Reinerton, Michigan, 2908","about":"Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n","registered":"2019-03-29T01:59:31 -01:00","latitude":45.193723,"longitude":-12.486778,"tags":["new issue","new issue","wontfix","wontfix"]} +{"id":37,"isActive":false,"balance":"$3,469.82","picture":"http://placehold.it/32x32","age":39,"color":"brown","name":"Opal Weiss","gender":"female","email":"opalweiss@chorizon.com","phone":"+1 (809) 400-3079","address":"535 Bogart Street, Frizzleburg, Arizona, 5222","about":"Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n","registered":"2019-09-04T07:22:28 -02:00","latitude":72.50376,"longitude":61.656435,"tags":["bug","bug","good first issue","good first issue"]} +{"id":38,"isActive":true,"balance":"$1,992.38","picture":"http://placehold.it/32x32","age":40,"color":"Green","name":"Christina Short","gender":"female","email":"christinashort@chorizon.com","phone":"+1 (884) 589-2705","address":"594 Willmohr Street, Dexter, Montana, 660","about":"Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n","registered":"2014-01-21T09:31:56 -01:00","latitude":-42.762739,"longitude":77.052349,"tags":["bug","new issue"]} +{"id":39,"isActive":false,"balance":"$1,722.85","picture":"http://placehold.it/32x32","age":29,"color":"brown","name":"Golden Horton","gender":"male","email":"goldenhorton@chorizon.com","phone":"+1 (903) 426-2489","address":"191 Schenck Avenue, Mayfair, North Dakota, 5000","about":"Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n","registered":"2015-08-19T02:56:41 -02:00","latitude":69.922534,"longitude":9.881433,"tags":["bug"]} +{"id":40,"isActive":false,"balance":"$1,656.54","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Stafford Emerson","gender":"male","email":"staffordemerson@chorizon.com","phone":"+1 (992) 455-2573","address":"523 Thornton Street, Conway, Vermont, 6331","about":"Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n","registered":"2019-02-16T04:07:08 -01:00","latitude":-29.143111,"longitude":-57.207703,"tags":["wontfix","good first issue","good first issue"]} +{"id":41,"isActive":false,"balance":"$1,861.56","picture":"http://placehold.it/32x32","age":21,"color":"brown","name":"Salinas Gamble","gender":"male","email":"salinasgamble@chorizon.com","phone":"+1 (901) 525-2373","address":"991 Nostrand Avenue, Kansas, Mississippi, 6756","about":"Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n","registered":"2017-08-21T05:47:53 -02:00","latitude":-22.593819,"longitude":-63.613004,"tags":["good first issue","bug","bug","wontfix"]} +{"id":42,"isActive":true,"balance":"$3,179.74","picture":"http://placehold.it/32x32","age":34,"color":"brown","name":"Graciela Russell","gender":"female","email":"gracielarussell@chorizon.com","phone":"+1 (893) 464-3951","address":"361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713","about":"Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n","registered":"2015-05-18T09:52:56 -02:00","latitude":-14.634444,"longitude":12.931783,"tags":["wontfix","bug","wontfix"]} +{"id":43,"isActive":true,"balance":"$1,777.38","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Arnold Bender","gender":"male","email":"arnoldbender@chorizon.com","phone":"+1 (945) 581-3808","address":"781 Lorraine Street, Gallina, American Samoa, 1832","about":"Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n","registered":"2018-12-23T02:26:30 -01:00","latitude":41.208579,"longitude":51.948925,"tags":["bug","good first issue","good first issue","wontfix"]} +{"id":44,"isActive":true,"balance":"$2,893.45","picture":"http://placehold.it/32x32","age":22,"color":"Green","name":"Joni Spears","gender":"female","email":"jonispears@chorizon.com","phone":"+1 (916) 565-2124","address":"307 Harwood Place, Canterwood, Maryland, 2047","about":"Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n","registered":"2015-03-01T12:38:28 -01:00","latitude":8.19071,"longitude":146.323808,"tags":["wontfix","new issue","good first issue","good first issue"]} +{"id":45,"isActive":true,"balance":"$2,830.36","picture":"http://placehold.it/32x32","age":20,"color":"brown","name":"Irene Bennett","gender":"female","email":"irenebennett@chorizon.com","phone":"+1 (904) 431-2211","address":"353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686","about":"Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n","registered":"2018-04-17T05:18:51 -02:00","latitude":-36.435177,"longitude":-127.552573,"tags":["bug","wontfix"]} +{"id":46,"isActive":true,"balance":"$1,348.04","picture":"http://placehold.it/32x32","age":34,"color":"Green","name":"Lawson Curtis","gender":"male","email":"lawsoncurtis@chorizon.com","phone":"+1 (896) 532-2172","address":"942 Gerritsen Avenue, Southmont, Kansas, 8915","about":"Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n","registered":"2016-08-23T01:41:09 -02:00","latitude":-48.783539,"longitude":20.492944,"tags":[]} +{"id":47,"isActive":true,"balance":"$1,132.41","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goff May","gender":"male","email":"goffmay@chorizon.com","phone":"+1 (859) 453-3415","address":"225 Rutledge Street, Boonville, Massachusetts, 4081","about":"Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n","registered":"2014-10-25T07:32:30 -02:00","latitude":13.079225,"longitude":76.215086,"tags":["bug"]} +{"id":48,"isActive":true,"balance":"$1,201.87","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goodman Becker","gender":"male","email":"goodmanbecker@chorizon.com","phone":"+1 (825) 470-3437","address":"388 Seigel Street, Sisquoc, Kentucky, 8231","about":"Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n","registered":"2019-09-05T04:49:03 -02:00","latitude":-23.792094,"longitude":-13.621221,"tags":["bug","bug","wontfix","new issue"]} +{"id":49,"isActive":true,"balance":"$1,476.39","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Maureen Dale","gender":"female","email":"maureendale@chorizon.com","phone":"+1 (984) 538-3684","address":"817 Newton Street, Bannock, Wyoming, 1468","about":"Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n","registered":"2018-04-26T06:04:40 -02:00","latitude":-64.196802,"longitude":-117.396238,"tags":["wontfix"]} +{"id":50,"isActive":true,"balance":"$1,947.08","picture":"http://placehold.it/32x32","age":21,"color":"Green","name":"Guerra Mcintyre","gender":"male","email":"guerramcintyre@chorizon.com","phone":"+1 (951) 536-2043","address":"423 Lombardy Street, Stewart, West Virginia, 908","about":"Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n","registered":"2015-07-16T05:11:42 -02:00","latitude":79.733743,"longitude":-20.602356,"tags":["bug","good first issue","good first issue"]} +{"id":51,"isActive":true,"balance":"$2,960.90","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Key Cervantes","gender":"male","email":"keycervantes@chorizon.com","phone":"+1 (931) 474-3865","address":"410 Barbey Street, Vernon, Oregon, 2328","about":"Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n","registered":"2019-12-15T12:13:35 -01:00","latitude":47.627647,"longitude":117.049918,"tags":["new issue"]} +{"id":52,"isActive":false,"balance":"$1,884.02","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Karen Nelson","gender":"female","email":"karennelson@chorizon.com","phone":"+1 (993) 528-3607","address":"930 Frank Court, Dunbar, New York, 8810","about":"Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n","registered":"2014-06-23T09:21:44 -02:00","latitude":-59.059033,"longitude":76.565373,"tags":["new issue","bug"]} +{"id":53,"isActive":true,"balance":"$3,559.55","picture":"http://placehold.it/32x32","age":32,"color":"brown","name":"Caitlin Burnett","gender":"female","email":"caitlinburnett@chorizon.com","phone":"+1 (945) 480-2796","address":"516 Senator Street, Emory, Iowa, 4145","about":"In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n","registered":"2019-01-09T02:26:31 -01:00","latitude":-82.774237,"longitude":42.316194,"tags":["bug","good first issue"]} +{"id":54,"isActive":true,"balance":"$2,113.29","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Richards Walls","gender":"male","email":"richardswalls@chorizon.com","phone":"+1 (865) 517-2982","address":"959 Brightwater Avenue, Stevens, Nevada, 2968","about":"Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n","registered":"2014-09-25T06:51:22 -02:00","latitude":80.09202,"longitude":87.49759,"tags":["wontfix","wontfix","bug"]} +{"id":55,"isActive":true,"balance":"$1,977.66","picture":"http://placehold.it/32x32","age":36,"color":"brown","name":"Combs Stanley","gender":"male","email":"combsstanley@chorizon.com","phone":"+1 (827) 419-2053","address":"153 Beverley Road, Siglerville, South Carolina, 3666","about":"Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n","registered":"2019-08-22T07:53:15 -02:00","latitude":78.386181,"longitude":143.661058,"tags":[]} +{"id":56,"isActive":false,"balance":"$3,886.12","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Tucker Barry","gender":"male","email":"tuckerbarry@chorizon.com","phone":"+1 (808) 544-3433","address":"805 Jamaica Avenue, Cornfields, Minnesota, 3689","about":"Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n","registered":"2016-08-29T07:28:00 -02:00","latitude":71.701551,"longitude":9.903068,"tags":[]} +{"id":57,"isActive":false,"balance":"$1,844.56","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Kaitlin Conner","gender":"female","email":"kaitlinconner@chorizon.com","phone":"+1 (862) 467-2666","address":"501 Knight Court, Joppa, Rhode Island, 274","about":"Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n","registered":"2019-05-30T06:38:24 -02:00","latitude":15.613464,"longitude":171.965629,"tags":[]} +{"id":58,"isActive":true,"balance":"$2,876.10","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Mamie Fischer","gender":"female","email":"mamiefischer@chorizon.com","phone":"+1 (948) 545-3901","address":"599 Hunterfly Place, Haena, Georgia, 6005","about":"Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n","registered":"2019-05-27T05:07:10 -02:00","latitude":70.915079,"longitude":-48.813584,"tags":["bug","wontfix","wontfix","good first issue"]} +{"id":59,"isActive":true,"balance":"$1,921.58","picture":"http://placehold.it/32x32","age":31,"color":"Green","name":"Harper Carson","gender":"male","email":"harpercarson@chorizon.com","phone":"+1 (912) 430-3243","address":"883 Dennett Place, Knowlton, New Mexico, 9219","about":"Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n","registered":"2019-12-07T07:33:15 -01:00","latitude":-60.812605,"longitude":-27.129016,"tags":["bug","new issue"]} +{"id":60,"isActive":true,"balance":"$1,770.93","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Jody Herrera","gender":"female","email":"jodyherrera@chorizon.com","phone":"+1 (890) 583-3222","address":"261 Jay Street, Strykersville, Ohio, 9248","about":"Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n","registered":"2016-05-21T01:00:02 -02:00","latitude":-36.846586,"longitude":131.156223,"tags":[]} +{"id":61,"isActive":false,"balance":"$2,813.41","picture":"http://placehold.it/32x32","age":37,"color":"Green","name":"Charles Castillo","gender":"male","email":"charlescastillo@chorizon.com","phone":"+1 (934) 467-2108","address":"675 Morton Street, Rew, Pennsylvania, 137","about":"Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n","registered":"2019-06-10T02:54:22 -02:00","latitude":-16.423202,"longitude":-146.293752,"tags":["new issue","new issue"]} +{"id":62,"isActive":true,"balance":"$3,341.35","picture":"http://placehold.it/32x32","age":33,"color":"blue","name":"Estelle Ramirez","gender":"female","email":"estelleramirez@chorizon.com","phone":"+1 (816) 459-2073","address":"636 Nolans Lane, Camptown, California, 7794","about":"Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n","registered":"2015-02-14T01:05:50 -01:00","latitude":-46.591249,"longitude":-83.385587,"tags":["good first issue","bug"]} +{"id":63,"isActive":true,"balance":"$2,478.30","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Knowles Hebert","gender":"male","email":"knowleshebert@chorizon.com","phone":"+1 (819) 409-2308","address":"361 Kathleen Court, Gratton, Connecticut, 7254","about":"Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n","registered":"2016-03-08T08:34:52 -01:00","latitude":71.042482,"longitude":152.460406,"tags":["good first issue","wontfix"]} +{"id":64,"isActive":false,"balance":"$2,559.09","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Thelma Mckenzie","gender":"female","email":"thelmamckenzie@chorizon.com","phone":"+1 (941) 596-2777","address":"202 Leonard Street, Riverton, Illinois, 8577","about":"Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n","registered":"2020-04-14T12:43:06 -02:00","latitude":16.026129,"longitude":105.464476,"tags":[]} +{"id":65,"isActive":true,"balance":"$1,025.08","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Carole Rowland","gender":"female","email":"carolerowland@chorizon.com","phone":"+1 (862) 558-3448","address":"941 Melba Court, Bluetown, Florida, 9555","about":"Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n","registered":"2014-12-01T05:55:35 -01:00","latitude":-0.191998,"longitude":43.389652,"tags":["wontfix"]} +{"id":66,"isActive":true,"balance":"$1,061.49","picture":"http://placehold.it/32x32","age":35,"color":"brown","name":"Higgins Aguilar","gender":"male","email":"higginsaguilar@chorizon.com","phone":"+1 (911) 540-3791","address":"132 Sackman Street, Layhill, Guam, 8729","about":"Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n","registered":"2015-04-05T02:10:07 -02:00","latitude":74.702813,"longitude":151.314972,"tags":["bug"]} +{"id":67,"isActive":true,"balance":"$3,510.14","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Ilene Gillespie","gender":"female","email":"ilenegillespie@chorizon.com","phone":"+1 (937) 575-2676","address":"835 Lake Street, Naomi, Alabama, 4131","about":"Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n","registered":"2015-06-28T09:41:45 -02:00","latitude":71.573342,"longitude":-95.295989,"tags":["wontfix","wontfix"]} +{"id":68,"isActive":false,"balance":"$1,539.98","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Angelina Dyer","gender":"female","email":"angelinadyer@chorizon.com","phone":"+1 (948) 574-3949","address":"575 Division Place, Gorham, Louisiana, 3458","about":"Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n","registered":"2014-07-08T06:34:36 -02:00","latitude":-85.649593,"longitude":66.126018,"tags":["good first issue"]} +{"id":69,"isActive":true,"balance":"$3,367.69","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Marks Burt","gender":"male","email":"marksburt@chorizon.com","phone":"+1 (895) 497-3138","address":"819 Village Road, Wadsworth, Delaware, 6099","about":"Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n","registered":"2014-08-31T06:12:18 -02:00","latitude":26.854112,"longitude":-143.313948,"tags":["good first issue"]} +{"id":70,"isActive":false,"balance":"$3,755.72","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Glass Perkins","gender":"male","email":"glassperkins@chorizon.com","phone":"+1 (923) 486-3725","address":"899 Roosevelt Court, Belleview, Idaho, 1737","about":"Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n","registered":"2015-05-22T05:44:33 -02:00","latitude":54.27147,"longitude":-65.065604,"tags":["wontfix"]} +{"id":71,"isActive":true,"balance":"$3,381.63","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Candace Sawyer","gender":"female","email":"candacesawyer@chorizon.com","phone":"+1 (830) 404-2636","address":"334 Arkansas Drive, Bordelonville, Tennessee, 8449","about":"Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n","registered":"2014-04-04T08:45:00 -02:00","latitude":6.484262,"longitude":-37.054928,"tags":["new issue","new issue"]} +{"id":72,"isActive":true,"balance":"$1,640.98","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Hendricks Martinez","gender":"male","email":"hendricksmartinez@chorizon.com","phone":"+1 (857) 566-3245","address":"636 Agate Court, Newry, Utah, 3304","about":"Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n","registered":"2018-06-15T10:36:11 -02:00","latitude":86.746034,"longitude":10.347893,"tags":["new issue"]} +{"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]} +{"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]} +{"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]} +{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]} \ No newline at end of file diff --git a/tests/assets/dumps/v1/test/settings.json b/tests/assets/dumps/v1/test/settings.json new file mode 100644 index 000000000..918cfab53 --- /dev/null +++ b/tests/assets/dumps/v1/test/settings.json @@ -0,0 +1,59 @@ +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": "email", + "searchableAttributes": [ + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "displayedAttributes": [ + "id", + "isActive", + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "stopWords": [ + "in", + "ad" + ], + "synonyms": { + "wolverine": ["xmen", "logan"], + "logan": ["wolverine", "xmen"] + }, + "attributesForFaceting": [ + "gender", + "color", + "tags", + "isActive" + ] +} diff --git a/tests/assets/dumps/v1/test/updates.jsonl b/tests/assets/dumps/v1/test/updates.jsonl new file mode 100644 index 000000000..0dcffdce0 --- /dev/null +++ b/tests/assets/dumps/v1/test/updates.jsonl @@ -0,0 +1,2 @@ +{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}} +{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} \ No newline at end of file diff --git a/tests/assets/test_set.json b/tests/assets/test_set.json new file mode 100644 index 000000000..63534c896 --- /dev/null +++ b/tests/assets/test_set.json @@ -0,0 +1,1613 @@ +[ + { + "id": 0, + "isActive": false, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ] + }, + { + "id": 1, + "isActive": true, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ] + }, + { + "id": 2, + "isActive": true, + "balance": "$2,467.47", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", + "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", + "registered": "2014-10-28T12:59:30 -01:00", + "latitude": -64.008555, + "longitude": 11.867098, + "tags": [ + "good first issue" + ] + }, + { + "id": 3, + "isActive": true, + "balance": "$3,344.40", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Adeline Flynn", + "gender": "female", + "email": "adelineflynn@chorizon.com", + "phone": "+1 (994) 600-2840", + "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948", + "about": "Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n", + "registered": "2014-03-27T06:24:45 -01:00", + "latitude": -74.485173, + "longitude": -11.059859, + "tags": [ + "bug", + "good first issue", + "wontfix", + "new issue" + ] + }, + { + "id": 4, + "isActive": false, + "balance": "$2,575.78", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "Green", + "name": "Mariana Pacheco", + "gender": "female", + "email": "marianapacheco@chorizon.com", + "phone": "+1 (820) 414-2223", + "address": "664 Rapelye Street, Faywood, California, 7320", + "about": "Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n", + "registered": "2015-09-02T03:23:35 -02:00", + "latitude": 75.763501, + "longitude": -78.777124, + "tags": [ + "new issue" + ] + }, + { + "id": 5, + "isActive": true, + "balance": "$3,793.09", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "Green", + "name": "Warren Watson", + "gender": "male", + "email": "warrenwatson@chorizon.com", + "phone": "+1 (807) 583-2427", + "address": "671 Prince Street, Faxon, Connecticut, 4275", + "about": "Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n", + "registered": "2017-06-04T06:02:17 -02:00", + "latitude": 29.979223, + "longitude": 25.358943, + "tags": [ + "wontfix", + "wontfix", + "wontfix" + ] + }, + { + "id": 6, + "isActive": true, + "balance": "$2,919.70", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "blue", + "name": "Shelia Berry", + "gender": "female", + "email": "sheliaberry@chorizon.com", + "phone": "+1 (853) 511-2651", + "address": "437 Forrest Street, Coventry, Illinois, 2056", + "about": "Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n", + "registered": "2018-07-11T02:45:01 -02:00", + "latitude": 54.815991, + "longitude": -118.690609, + "tags": [ + "good first issue", + "bug", + "wontfix", + "new issue" + ] + }, + { + "id": 7, + "isActive": true, + "balance": "$1,349.50", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Chrystal Boyd", + "gender": "female", + "email": "chrystalboyd@chorizon.com", + "phone": "+1 (936) 563-2802", + "address": "670 Croton Loop, Sussex, Florida, 4692", + "about": "Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n", + "registered": "2016-11-01T07:36:04 -01:00", + "latitude": -24.711933, + "longitude": 147.246705, + "tags": [] + }, + { + "id": 8, + "isActive": false, + "balance": "$3,999.56", + "picture": "http://placehold.it/32x32", + "age": 30, + "color": "brown", + "name": "Martin Porter", + "gender": "male", + "email": "martinporter@chorizon.com", + "phone": "+1 (895) 580-2304", + "address": "577 Regent Place, Aguila, Guam, 6554", + "about": "Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n", + "registered": "2014-09-20T02:08:30 -02:00", + "latitude": -88.344273, + "longitude": 37.964466, + "tags": [] + }, + { + "id": 9, + "isActive": true, + "balance": "$3,729.71", + "picture": "http://placehold.it/32x32", + "age": 26, + "color": "blue", + "name": "Kelli Mendez", + "gender": "female", + "email": "kellimendez@chorizon.com", + "phone": "+1 (936) 401-2236", + "address": "242 Caton Place, Grazierville, Alabama, 3968", + "about": "Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n", + "registered": "2018-05-04T10:35:30 -02:00", + "latitude": 49.37551, + "longitude": 41.872323, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 10, + "isActive": false, + "balance": "$1,127.47", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "blue", + "name": "Maddox Johns", + "gender": "male", + "email": "maddoxjohns@chorizon.com", + "phone": "+1 (892) 470-2357", + "address": "756 Beard Street, Avalon, Louisiana, 114", + "about": "Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n", + "registered": "2016-04-22T06:41:25 -02:00", + "latitude": 66.640229, + "longitude": -17.222666, + "tags": [ + "new issue", + "good first issue", + "good first issue", + "new issue" + ] + }, + { + "id": 11, + "isActive": true, + "balance": "$1,351.43", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Evans Wagner", + "gender": "male", + "email": "evanswagner@chorizon.com", + "phone": "+1 (889) 496-2332", + "address": "118 Monaco Place, Lutsen, Delaware, 6209", + "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", + "registered": "2016-10-27T01:26:31 -02:00", + "latitude": -77.673222, + "longitude": -142.657214, + "tags": [ + "good first issue", + "good first issue" + ] + }, + { + "id": 12, + "isActive": false, + "balance": "$3,394.96", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "blue", + "name": "Aida Kirby", + "gender": "female", + "email": "aidakirby@chorizon.com", + "phone": "+1 (942) 532-2325", + "address": "797 Engert Avenue, Wilsonia, Idaho, 6532", + "about": "Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n", + "registered": "2018-06-18T04:39:57 -02:00", + "latitude": -58.062041, + "longitude": 34.999254, + "tags": [ + "new issue", + "wontfix", + "bug", + "new issue" + ] + }, + { + "id": 13, + "isActive": true, + "balance": "$2,812.62", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "blue", + "name": "Nelda Burris", + "gender": "female", + "email": "neldaburris@chorizon.com", + "phone": "+1 (813) 600-2576", + "address": "160 Opal Court, Fowlerville, Tennessee, 2170", + "about": "Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n", + "registered": "2015-08-15T12:39:53 -02:00", + "latitude": 66.6871, + "longitude": 179.549488, + "tags": [ + "wontfix" + ] + }, + { + "id": 14, + "isActive": true, + "balance": "$1,718.33", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Jennifer Hart", + "gender": "female", + "email": "jenniferhart@chorizon.com", + "phone": "+1 (850) 537-2513", + "address": "124 Veranda Place, Nash, Utah, 985", + "about": "Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n", + "registered": "2016-09-04T11:46:59 -02:00", + "latitude": -66.827751, + "longitude": 99.220079, + "tags": [ + "wontfix", + "bug", + "new issue", + "new issue" + ] + }, + { + "id": 15, + "isActive": false, + "balance": "$2,698.16", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "blue", + "name": "Aurelia Contreras", + "gender": "female", + "email": "aureliacontreras@chorizon.com", + "phone": "+1 (932) 442-3103", + "address": "655 Dwight Street, Grapeview, Palau, 8356", + "about": "Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n", + "registered": "2014-09-11T10:43:15 -02:00", + "latitude": -71.328973, + "longitude": 133.404895, + "tags": [ + "wontfix", + "bug", + "good first issue" + ] + }, + { + "id": 16, + "isActive": true, + "balance": "$3,303.25", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Estella Bass", + "gender": "female", + "email": "estellabass@chorizon.com", + "phone": "+1 (825) 436-2909", + "address": "435 Rockwell Place, Garberville, Wisconsin, 2230", + "about": "Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n", + "registered": "2017-11-23T09:32:09 -01:00", + "latitude": 81.17014, + "longitude": -145.262693, + "tags": [ + "new issue" + ] + }, + { + "id": 17, + "isActive": false, + "balance": "$3,579.20", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "brown", + "name": "Ortega Brennan", + "gender": "male", + "email": "ortegabrennan@chorizon.com", + "phone": "+1 (906) 526-2287", + "address": "440 Berry Street, Rivera, Maine, 1849", + "about": "Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n", + "registered": "2016-03-31T02:17:13 -02:00", + "latitude": -68.407524, + "longitude": -113.642067, + "tags": [ + "new issue", + "wontfix" + ] + }, + { + "id": 18, + "isActive": false, + "balance": "$1,484.92", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "blue", + "name": "Leonard Tillman", + "gender": "male", + "email": "leonardtillman@chorizon.com", + "phone": "+1 (864) 541-3456", + "address": "985 Provost Street, Charco, New Hampshire, 8632", + "about": "Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n", + "registered": "2018-05-06T08:21:27 -02:00", + "latitude": -8.581801, + "longitude": -61.910062, + "tags": [ + "wontfix", + "new issue", + "bug", + "bug" + ] + }, + { + "id": 19, + "isActive": true, + "balance": "$3,572.55", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Dale Payne", + "gender": "male", + "email": "dalepayne@chorizon.com", + "phone": "+1 (814) 469-3499", + "address": "536 Dare Court, Ironton, Arkansas, 8605", + "about": "Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n", + "registered": "2019-10-11T01:01:33 -02:00", + "latitude": -18.280968, + "longitude": -126.091797, + "tags": [ + "bug", + "wontfix", + "wontfix", + "wontfix" + ] + }, + { + "id": 20, + "isActive": true, + "balance": "$1,986.48", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Florence Long", + "gender": "female", + "email": "florencelong@chorizon.com", + "phone": "+1 (972) 557-3858", + "address": "519 Hendrickson Street, Templeton, Hawaii, 2389", + "about": "Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n", + "registered": "2016-05-02T09:18:59 -02:00", + "latitude": -27.110866, + "longitude": -45.09445, + "tags": [] + }, + { + "id": 21, + "isActive": true, + "balance": "$1,440.09", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "blue", + "name": "Levy Whitley", + "gender": "male", + "email": "levywhitley@chorizon.com", + "phone": "+1 (911) 458-2411", + "address": "187 Thomas Street, Hachita, North Carolina, 2989", + "about": "Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n", + "registered": "2014-04-30T07:31:38 -02:00", + "latitude": -6.537315, + "longitude": 171.813536, + "tags": [ + "bug" + ] + }, + { + "id": 22, + "isActive": false, + "balance": "$2,938.57", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Bernard Mcfarland", + "gender": "male", + "email": "bernardmcfarland@chorizon.com", + "phone": "+1 (979) 442-3386", + "address": "409 Hall Street, Keyport, Federated States Of Micronesia, 7011", + "about": "Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n", + "registered": "2017-08-10T10:07:59 -02:00", + "latitude": 63.766795, + "longitude": 68.177069, + "tags": [] + }, + { + "id": 23, + "isActive": true, + "balance": "$1,678.49", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "brown", + "name": "Blanca Mcclain", + "gender": "female", + "email": "blancamcclain@chorizon.com", + "phone": "+1 (976) 439-2772", + "address": "176 Crooke Avenue, Valle, Virginia, 5373", + "about": "Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n", + "registered": "2015-10-12T11:57:28 -02:00", + "latitude": -8.944564, + "longitude": -150.711709, + "tags": [ + "bug", + "wontfix", + "good first issue" + ] + }, + { + "id": 24, + "isActive": true, + "balance": "$2,276.87", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Espinoza Ford", + "gender": "male", + "email": "espinozaford@chorizon.com", + "phone": "+1 (945) 429-3975", + "address": "137 Bowery Street, Itmann, District Of Columbia, 1864", + "about": "Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n", + "registered": "2014-03-26T02:16:08 -01:00", + "latitude": -37.137666, + "longitude": -51.811757, + "tags": [ + "wontfix", + "bug" + ] + }, + { + "id": 25, + "isActive": true, + "balance": "$3,973.43", + "picture": "http://placehold.it/32x32", + "age": 29, + "color": "Green", + "name": "Sykes Conley", + "gender": "male", + "email": "sykesconley@chorizon.com", + "phone": "+1 (851) 401-3916", + "address": "345 Grand Street, Woodlands, Missouri, 4461", + "about": "Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n", + "registered": "2015-09-12T06:03:56 -02:00", + "latitude": 67.282955, + "longitude": -64.341323, + "tags": [ + "wontfix" + ] + }, + { + "id": 26, + "isActive": false, + "balance": "$1,431.50", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Barlow Duran", + "gender": "male", + "email": "barlowduran@chorizon.com", + "phone": "+1 (995) 436-2562", + "address": "481 Everett Avenue, Allison, Nebraska, 3065", + "about": "Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n", + "registered": "2017-06-29T04:28:43 -02:00", + "latitude": -38.70606, + "longitude": 55.02816, + "tags": [ + "new issue" + ] + }, + { + "id": 27, + "isActive": true, + "balance": "$3,478.27", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "blue", + "name": "Schwartz Morgan", + "gender": "male", + "email": "schwartzmorgan@chorizon.com", + "phone": "+1 (861) 507-2067", + "address": "451 Lincoln Road, Fairlee, Washington, 2717", + "about": "Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n", + "registered": "2016-05-10T08:34:54 -02:00", + "latitude": -75.886403, + "longitude": 93.044471, + "tags": [ + "bug", + "bug", + "wontfix", + "wontfix" + ] + }, + { + "id": 28, + "isActive": true, + "balance": "$2,825.59", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Kristy Leon", + "gender": "female", + "email": "kristyleon@chorizon.com", + "phone": "+1 (948) 465-2563", + "address": "594 Macon Street, Floris, South Dakota, 3565", + "about": "Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n", + "registered": "2014-12-14T04:10:29 -01:00", + "latitude": -50.01615, + "longitude": -68.908804, + "tags": [ + "wontfix", + "good first issue" + ] + }, + { + "id": 29, + "isActive": false, + "balance": "$3,028.03", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "blue", + "name": "Ashley Pittman", + "gender": "male", + "email": "ashleypittman@chorizon.com", + "phone": "+1 (928) 507-3523", + "address": "646 Adelphi Street, Clara, Colorado, 6056", + "about": "Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n", + "registered": "2016-01-07T10:40:48 -01:00", + "latitude": -58.766037, + "longitude": -124.828485, + "tags": [ + "wontfix" + ] + }, + { + "id": 30, + "isActive": true, + "balance": "$2,021.11", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Stacy Espinoza", + "gender": "female", + "email": "stacyespinoza@chorizon.com", + "phone": "+1 (999) 487-3253", + "address": "931 Alabama Avenue, Bangor, Alaska, 8215", + "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", + "registered": "2014-07-16T06:15:53 -02:00", + "latitude": 41.560197, + "longitude": 177.697, + "tags": [ + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 31, + "isActive": false, + "balance": "$3,609.82", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Vilma Garza", + "gender": "female", + "email": "vilmagarza@chorizon.com", + "phone": "+1 (944) 585-2021", + "address": "565 Tech Place, Sedley, Puerto Rico, 858", + "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", + "registered": "2017-06-30T07:43:52 -02:00", + "latitude": -12.574889, + "longitude": -54.771186, + "tags": [ + "new issue", + "wontfix", + "wontfix" + ] + }, + { + "id": 32, + "isActive": false, + "balance": "$2,882.34", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "brown", + "name": "June Dunlap", + "gender": "female", + "email": "junedunlap@chorizon.com", + "phone": "+1 (997) 504-2937", + "address": "353 Cozine Avenue, Goodville, Indiana, 1438", + "about": "Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n", + "registered": "2016-08-23T08:54:11 -02:00", + "latitude": -27.883363, + "longitude": -163.919683, + "tags": [ + "new issue", + "new issue", + "bug", + "wontfix" + ] + }, + { + "id": 33, + "isActive": true, + "balance": "$3,556.54", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Cecilia Greer", + "gender": "female", + "email": "ceciliagreer@chorizon.com", + "phone": "+1 (977) 573-3498", + "address": "696 Withers Street, Lydia, Oklahoma, 3220", + "about": "Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n", + "registered": "2017-01-13T11:30:12 -01:00", + "latitude": 60.467215, + "longitude": 84.684575, + "tags": [ + "wontfix", + "good first issue", + "good first issue", + "wontfix" + ] + }, + { + "id": 34, + "isActive": true, + "balance": "$1,413.35", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Mckay Schroeder", + "gender": "male", + "email": "mckayschroeder@chorizon.com", + "phone": "+1 (816) 480-3657", + "address": "958 Miami Court, Rehrersburg, Northern Mariana Islands, 567", + "about": "Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n", + "registered": "2016-02-08T04:50:15 -01:00", + "latitude": -72.413287, + "longitude": -159.254371, + "tags": [ + "good first issue" + ] + }, + { + "id": 35, + "isActive": true, + "balance": "$2,306.53", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Sawyer Mccormick", + "gender": "male", + "email": "sawyermccormick@chorizon.com", + "phone": "+1 (829) 569-3012", + "address": "749 Apollo Street, Eastvale, Texas, 7373", + "about": "Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n", + "registered": "2019-11-30T11:53:23 -01:00", + "latitude": -48.978194, + "longitude": 110.950191, + "tags": [ + "good first issue", + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 36, + "isActive": false, + "balance": "$1,844.54", + "picture": "http://placehold.it/32x32", + "age": 37, + "color": "brown", + "name": "Barbra Valenzuela", + "gender": "female", + "email": "barbravalenzuela@chorizon.com", + "phone": "+1 (992) 512-2649", + "address": "617 Schenck Court, Reinerton, Michigan, 2908", + "about": "Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n", + "registered": "2019-03-29T01:59:31 -01:00", + "latitude": 45.193723, + "longitude": -12.486778, + "tags": [ + "new issue", + "new issue", + "wontfix", + "wontfix" + ] + }, + { + "id": 37, + "isActive": false, + "balance": "$3,469.82", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "brown", + "name": "Opal Weiss", + "gender": "female", + "email": "opalweiss@chorizon.com", + "phone": "+1 (809) 400-3079", + "address": "535 Bogart Street, Frizzleburg, Arizona, 5222", + "about": "Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n", + "registered": "2019-09-04T07:22:28 -02:00", + "latitude": 72.50376, + "longitude": 61.656435, + "tags": [ + "bug", + "bug", + "good first issue", + "good first issue" + ] + }, + { + "id": 38, + "isActive": true, + "balance": "$1,992.38", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "Green", + "name": "Christina Short", + "gender": "female", + "email": "christinashort@chorizon.com", + "phone": "+1 (884) 589-2705", + "address": "594 Willmohr Street, Dexter, Montana, 660", + "about": "Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n", + "registered": "2014-01-21T09:31:56 -01:00", + "latitude": -42.762739, + "longitude": 77.052349, + "tags": [ + "bug", + "new issue" + ] + }, + { + "id": 39, + "isActive": false, + "balance": "$1,722.85", + "picture": "http://placehold.it/32x32", + "age": 29, + "color": "brown", + "name": "Golden Horton", + "gender": "male", + "email": "goldenhorton@chorizon.com", + "phone": "+1 (903) 426-2489", + "address": "191 Schenck Avenue, Mayfair, North Dakota, 5000", + "about": "Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n", + "registered": "2015-08-19T02:56:41 -02:00", + "latitude": 69.922534, + "longitude": 9.881433, + "tags": [ + "bug" + ] + }, + { + "id": 40, + "isActive": false, + "balance": "$1,656.54", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "blue", + "name": "Stafford Emerson", + "gender": "male", + "email": "staffordemerson@chorizon.com", + "phone": "+1 (992) 455-2573", + "address": "523 Thornton Street, Conway, Vermont, 6331", + "about": "Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n", + "registered": "2019-02-16T04:07:08 -01:00", + "latitude": -29.143111, + "longitude": -57.207703, + "tags": [ + "wontfix", + "good first issue", + "good first issue" + ] + }, + { + "id": 41, + "isActive": false, + "balance": "$1,861.56", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "brown", + "name": "Salinas Gamble", + "gender": "male", + "email": "salinasgamble@chorizon.com", + "phone": "+1 (901) 525-2373", + "address": "991 Nostrand Avenue, Kansas, Mississippi, 6756", + "about": "Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n", + "registered": "2017-08-21T05:47:53 -02:00", + "latitude": -22.593819, + "longitude": -63.613004, + "tags": [ + "good first issue", + "bug", + "bug", + "wontfix" + ] + }, + { + "id": 42, + "isActive": true, + "balance": "$3,179.74", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "brown", + "name": "Graciela Russell", + "gender": "female", + "email": "gracielarussell@chorizon.com", + "phone": "+1 (893) 464-3951", + "address": "361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713", + "about": "Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n", + "registered": "2015-05-18T09:52:56 -02:00", + "latitude": -14.634444, + "longitude": 12.931783, + "tags": [ + "wontfix", + "bug", + "wontfix" + ] + }, + { + "id": 43, + "isActive": true, + "balance": "$1,777.38", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "blue", + "name": "Arnold Bender", + "gender": "male", + "email": "arnoldbender@chorizon.com", + "phone": "+1 (945) 581-3808", + "address": "781 Lorraine Street, Gallina, American Samoa, 1832", + "about": "Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n", + "registered": "2018-12-23T02:26:30 -01:00", + "latitude": 41.208579, + "longitude": 51.948925, + "tags": [ + "bug", + "good first issue", + "good first issue", + "wontfix" + ] + }, + { + "id": 44, + "isActive": true, + "balance": "$2,893.45", + "picture": "http://placehold.it/32x32", + "age": 22, + "color": "Green", + "name": "Joni Spears", + "gender": "female", + "email": "jonispears@chorizon.com", + "phone": "+1 (916) 565-2124", + "address": "307 Harwood Place, Canterwood, Maryland, 2047", + "about": "Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n", + "registered": "2015-03-01T12:38:28 -01:00", + "latitude": 8.19071, + "longitude": 146.323808, + "tags": [ + "wontfix", + "new issue", + "good first issue", + "good first issue" + ] + }, + { + "id": 45, + "isActive": true, + "balance": "$2,830.36", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "brown", + "name": "Irene Bennett", + "gender": "female", + "email": "irenebennett@chorizon.com", + "phone": "+1 (904) 431-2211", + "address": "353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686", + "about": "Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n", + "registered": "2018-04-17T05:18:51 -02:00", + "latitude": -36.435177, + "longitude": -127.552573, + "tags": [ + "bug", + "wontfix" + ] + }, + { + "id": 46, + "isActive": true, + "balance": "$1,348.04", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "Green", + "name": "Lawson Curtis", + "gender": "male", + "email": "lawsoncurtis@chorizon.com", + "phone": "+1 (896) 532-2172", + "address": "942 Gerritsen Avenue, Southmont, Kansas, 8915", + "about": "Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n", + "registered": "2016-08-23T01:41:09 -02:00", + "latitude": -48.783539, + "longitude": 20.492944, + "tags": [] + }, + { + "id": 47, + "isActive": true, + "balance": "$1,132.41", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Goff May", + "gender": "male", + "email": "goffmay@chorizon.com", + "phone": "+1 (859) 453-3415", + "address": "225 Rutledge Street, Boonville, Massachusetts, 4081", + "about": "Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n", + "registered": "2014-10-25T07:32:30 -02:00", + "latitude": 13.079225, + "longitude": 76.215086, + "tags": [ + "bug" + ] + }, + { + "id": 48, + "isActive": true, + "balance": "$1,201.87", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Goodman Becker", + "gender": "male", + "email": "goodmanbecker@chorizon.com", + "phone": "+1 (825) 470-3437", + "address": "388 Seigel Street, Sisquoc, Kentucky, 8231", + "about": "Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n", + "registered": "2019-09-05T04:49:03 -02:00", + "latitude": -23.792094, + "longitude": -13.621221, + "tags": [ + "bug", + "bug", + "wontfix", + "new issue" + ] + }, + { + "id": 49, + "isActive": true, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ] + }, + { + "id": 50, + "isActive": true, + "balance": "$1,947.08", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "Green", + "name": "Guerra Mcintyre", + "gender": "male", + "email": "guerramcintyre@chorizon.com", + "phone": "+1 (951) 536-2043", + "address": "423 Lombardy Street, Stewart, West Virginia, 908", + "about": "Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n", + "registered": "2015-07-16T05:11:42 -02:00", + "latitude": 79.733743, + "longitude": -20.602356, + "tags": [ + "bug", + "good first issue", + "good first issue" + ] + }, + { + "id": 51, + "isActive": true, + "balance": "$2,960.90", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "blue", + "name": "Key Cervantes", + "gender": "male", + "email": "keycervantes@chorizon.com", + "phone": "+1 (931) 474-3865", + "address": "410 Barbey Street, Vernon, Oregon, 2328", + "about": "Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n", + "registered": "2019-12-15T12:13:35 -01:00", + "latitude": 47.627647, + "longitude": 117.049918, + "tags": [ + "new issue" + ] + }, + { + "id": 52, + "isActive": false, + "balance": "$1,884.02", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Karen Nelson", + "gender": "female", + "email": "karennelson@chorizon.com", + "phone": "+1 (993) 528-3607", + "address": "930 Frank Court, Dunbar, New York, 8810", + "about": "Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n", + "registered": "2014-06-23T09:21:44 -02:00", + "latitude": -59.059033, + "longitude": 76.565373, + "tags": [ + "new issue", + "bug" + ] + }, + { + "id": 53, + "isActive": true, + "balance": "$3,559.55", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "brown", + "name": "Caitlin Burnett", + "gender": "female", + "email": "caitlinburnett@chorizon.com", + "phone": "+1 (945) 480-2796", + "address": "516 Senator Street, Emory, Iowa, 4145", + "about": "In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n", + "registered": "2019-01-09T02:26:31 -01:00", + "latitude": -82.774237, + "longitude": 42.316194, + "tags": [ + "bug", + "good first issue" + ] + }, + { + "id": 54, + "isActive": true, + "balance": "$2,113.29", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Richards Walls", + "gender": "male", + "email": "richardswalls@chorizon.com", + "phone": "+1 (865) 517-2982", + "address": "959 Brightwater Avenue, Stevens, Nevada, 2968", + "about": "Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n", + "registered": "2014-09-25T06:51:22 -02:00", + "latitude": 80.09202, + "longitude": 87.49759, + "tags": [ + "wontfix", + "wontfix", + "bug" + ] + }, + { + "id": 55, + "isActive": true, + "balance": "$1,977.66", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "brown", + "name": "Combs Stanley", + "gender": "male", + "email": "combsstanley@chorizon.com", + "phone": "+1 (827) 419-2053", + "address": "153 Beverley Road, Siglerville, South Carolina, 3666", + "about": "Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n", + "registered": "2019-08-22T07:53:15 -02:00", + "latitude": 78.386181, + "longitude": 143.661058, + "tags": [] + }, + { + "id": 56, + "isActive": false, + "balance": "$3,886.12", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "brown", + "name": "Tucker Barry", + "gender": "male", + "email": "tuckerbarry@chorizon.com", + "phone": "+1 (808) 544-3433", + "address": "805 Jamaica Avenue, Cornfields, Minnesota, 3689", + "about": "Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n", + "registered": "2016-08-29T07:28:00 -02:00", + "latitude": 71.701551, + "longitude": 9.903068, + "tags": [] + }, + { + "id": 57, + "isActive": false, + "balance": "$1,844.56", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "Green", + "name": "Kaitlin Conner", + "gender": "female", + "email": "kaitlinconner@chorizon.com", + "phone": "+1 (862) 467-2666", + "address": "501 Knight Court, Joppa, Rhode Island, 274", + "about": "Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n", + "registered": "2019-05-30T06:38:24 -02:00", + "latitude": 15.613464, + "longitude": 171.965629, + "tags": [] + }, + { + "id": 58, + "isActive": true, + "balance": "$2,876.10", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Mamie Fischer", + "gender": "female", + "email": "mamiefischer@chorizon.com", + "phone": "+1 (948) 545-3901", + "address": "599 Hunterfly Place, Haena, Georgia, 6005", + "about": "Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n", + "registered": "2019-05-27T05:07:10 -02:00", + "latitude": 70.915079, + "longitude": -48.813584, + "tags": [ + "bug", + "wontfix", + "wontfix", + "good first issue" + ] + }, + { + "id": 59, + "isActive": true, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ] + }, + { + "id": 60, + "isActive": true, + "balance": "$1,770.93", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "brown", + "name": "Jody Herrera", + "gender": "female", + "email": "jodyherrera@chorizon.com", + "phone": "+1 (890) 583-3222", + "address": "261 Jay Street, Strykersville, Ohio, 9248", + "about": "Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n", + "registered": "2016-05-21T01:00:02 -02:00", + "latitude": -36.846586, + "longitude": 131.156223, + "tags": [] + }, + { + "id": 61, + "isActive": false, + "balance": "$2,813.41", + "picture": "http://placehold.it/32x32", + "age": 37, + "color": "Green", + "name": "Charles Castillo", + "gender": "male", + "email": "charlescastillo@chorizon.com", + "phone": "+1 (934) 467-2108", + "address": "675 Morton Street, Rew, Pennsylvania, 137", + "about": "Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n", + "registered": "2019-06-10T02:54:22 -02:00", + "latitude": -16.423202, + "longitude": -146.293752, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 62, + "isActive": true, + "balance": "$3,341.35", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "blue", + "name": "Estelle Ramirez", + "gender": "female", + "email": "estelleramirez@chorizon.com", + "phone": "+1 (816) 459-2073", + "address": "636 Nolans Lane, Camptown, California, 7794", + "about": "Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n", + "registered": "2015-02-14T01:05:50 -01:00", + "latitude": -46.591249, + "longitude": -83.385587, + "tags": [ + "good first issue", + "bug" + ] + }, + { + "id": 63, + "isActive": true, + "balance": "$2,478.30", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "blue", + "name": "Knowles Hebert", + "gender": "male", + "email": "knowleshebert@chorizon.com", + "phone": "+1 (819) 409-2308", + "address": "361 Kathleen Court, Gratton, Connecticut, 7254", + "about": "Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n", + "registered": "2016-03-08T08:34:52 -01:00", + "latitude": 71.042482, + "longitude": 152.460406, + "tags": [ + "good first issue", + "wontfix" + ] + }, + { + "id": 64, + "isActive": false, + "balance": "$2,559.09", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Thelma Mckenzie", + "gender": "female", + "email": "thelmamckenzie@chorizon.com", + "phone": "+1 (941) 596-2777", + "address": "202 Leonard Street, Riverton, Illinois, 8577", + "about": "Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n", + "registered": "2020-04-14T12:43:06 -02:00", + "latitude": 16.026129, + "longitude": 105.464476, + "tags": [] + }, + { + "id": 65, + "isActive": true, + "balance": "$1,025.08", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Carole Rowland", + "gender": "female", + "email": "carolerowland@chorizon.com", + "phone": "+1 (862) 558-3448", + "address": "941 Melba Court, Bluetown, Florida, 9555", + "about": "Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n", + "registered": "2014-12-01T05:55:35 -01:00", + "latitude": -0.191998, + "longitude": 43.389652, + "tags": [ + "wontfix" + ] + }, + { + "id": 66, + "isActive": true, + "balance": "$1,061.49", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "brown", + "name": "Higgins Aguilar", + "gender": "male", + "email": "higginsaguilar@chorizon.com", + "phone": "+1 (911) 540-3791", + "address": "132 Sackman Street, Layhill, Guam, 8729", + "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", + "registered": "2015-04-05T02:10:07 -02:00", + "latitude": 74.702813, + "longitude": 151.314972, + "tags": [ + "bug" + ] + }, + { + "id": 67, + "isActive": true, + "balance": "$3,510.14", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Ilene Gillespie", + "gender": "female", + "email": "ilenegillespie@chorizon.com", + "phone": "+1 (937) 575-2676", + "address": "835 Lake Street, Naomi, Alabama, 4131", + "about": "Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n", + "registered": "2015-06-28T09:41:45 -02:00", + "latitude": 71.573342, + "longitude": -95.295989, + "tags": [ + "wontfix", + "wontfix" + ] + }, + { + "id": 68, + "isActive": false, + "balance": "$1,539.98", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Angelina Dyer", + "gender": "female", + "email": "angelinadyer@chorizon.com", + "phone": "+1 (948) 574-3949", + "address": "575 Division Place, Gorham, Louisiana, 3458", + "about": "Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n", + "registered": "2014-07-08T06:34:36 -02:00", + "latitude": -85.649593, + "longitude": 66.126018, + "tags": [ + "good first issue" + ] + }, + { + "id": 69, + "isActive": true, + "balance": "$3,367.69", + "picture": "http://placehold.it/32x32", + "age": 30, + "color": "brown", + "name": "Marks Burt", + "gender": "male", + "email": "marksburt@chorizon.com", + "phone": "+1 (895) 497-3138", + "address": "819 Village Road, Wadsworth, Delaware, 6099", + "about": "Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n", + "registered": "2014-08-31T06:12:18 -02:00", + "latitude": 26.854112, + "longitude": -143.313948, + "tags": [ + "good first issue" + ] + }, + { + "id": 70, + "isActive": false, + "balance": "$3,755.72", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "blue", + "name": "Glass Perkins", + "gender": "male", + "email": "glassperkins@chorizon.com", + "phone": "+1 (923) 486-3725", + "address": "899 Roosevelt Court, Belleview, Idaho, 1737", + "about": "Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n", + "registered": "2015-05-22T05:44:33 -02:00", + "latitude": 54.27147, + "longitude": -65.065604, + "tags": [ + "wontfix" + ] + }, + { + "id": 71, + "isActive": true, + "balance": "$3,381.63", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Candace Sawyer", + "gender": "female", + "email": "candacesawyer@chorizon.com", + "phone": "+1 (830) 404-2636", + "address": "334 Arkansas Drive, Bordelonville, Tennessee, 8449", + "about": "Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n", + "registered": "2014-04-04T08:45:00 -02:00", + "latitude": 6.484262, + "longitude": -37.054928, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 72, + "isActive": true, + "balance": "$1,640.98", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Hendricks Martinez", + "gender": "male", + "email": "hendricksmartinez@chorizon.com", + "phone": "+1 (857) 566-3245", + "address": "636 Agate Court, Newry, Utah, 3304", + "about": "Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n", + "registered": "2018-06-15T10:36:11 -02:00", + "latitude": 86.746034, + "longitude": 10.347893, + "tags": [ + "new issue" + ] + }, + { + "id": 73, + "isActive": false, + "balance": "$1,239.74", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "blue", + "name": "Eleanor Shepherd", + "gender": "female", + "email": "eleanorshepherd@chorizon.com", + "phone": "+1 (894) 567-2617", + "address": "670 Lafayette Walk, Darlington, Palau, 8803", + "about": "Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n", + "registered": "2020-02-29T12:15:28 -01:00", + "latitude": 35.749621, + "longitude": -94.40842, + "tags": [ + "good first issue", + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 74, + "isActive": true, + "balance": "$1,180.90", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Stark Wong", + "gender": "male", + "email": "starkwong@chorizon.com", + "phone": "+1 (805) 575-3055", + "address": "522 Bond Street, Bawcomville, Wisconsin, 324", + "about": "Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n", + "registered": "2020-01-25T10:47:48 -01:00", + "latitude": -80.452139, + "longitude": 160.72546, + "tags": [ + "wontfix" + ] + }, + { + "id": 75, + "isActive": false, + "balance": "$1,913.42", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Emma Jacobs", + "gender": "female", + "email": "emmajacobs@chorizon.com", + "phone": "+1 (899) 554-3847", + "address": "173 Tapscott Street, Esmont, Maine, 7450", + "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", + "registered": "2019-03-29T06:24:13 -01:00", + "latitude": -35.53722, + "longitude": 155.703874, + "tags": [] + }, + { + "id": 76, + "isActive": false, + "balance": "$1,274.29", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "Green", + "name": "Clarice Gardner", + "gender": "female", + "email": "claricegardner@chorizon.com", + "phone": "+1 (810) 407-3258", + "address": "894 Brooklyn Road, Utting, New Hampshire, 6404", + "about": "Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n", + "registered": "2014-10-20T10:13:32 -02:00", + "latitude": 17.11935, + "longitude": 65.38197, + "tags": [ + "new issue", + "wontfix" + ] + } +] diff --git a/tests/common.rs b/tests/common.rs new file mode 100644 index 000000000..43cea1447 --- /dev/null +++ b/tests/common.rs @@ -0,0 +1,569 @@ +#![allow(dead_code)] + +use actix_web::{http::StatusCode, test}; +use serde_json::{json, Value}; +use std::time::Duration; +use tempdir::TempDir; +use tokio::time::delay_for; + +use meilisearch_core::DatabaseOptions; +use meilisearch_http::data::Data; +use meilisearch_http::helpers::NormalizePath; +use meilisearch_http::option::Opt; + +/// Performs a search test on both post and get routes +#[macro_export] +macro_rules! test_post_get_search { + ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => { + let post_query: meilisearch_http::routes::search::SearchQueryPost = + serde_json::from_str(&$query.clone().to_string()).unwrap(); + let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into(); + let get_query = ::serde_url_params::to_string(&get_query).unwrap(); + let ($response, $status_code) = $server.search_get(&get_query).await; + let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { + panic!( + "panic in get route: {:?}", + e.downcast_ref::<&str>().unwrap() + ) + }); + let ($response, $status_code) = $server.search_post($query).await; + let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { + panic!( + "panic in post route: {:?}", + e.downcast_ref::<&str>().unwrap() + ) + }); + }; +} + +pub struct Server { + pub uid: String, + pub data: Data, +} + +impl Server { + pub fn with_uid(uid: &str) -> Server { + let tmp_dir = TempDir::new("meilisearch").unwrap(); + + let default_db_options = DatabaseOptions::default(); + + let opt = Opt { + db_path: tmp_dir.path().join("db").to_str().unwrap().to_string(), + dumps_dir: tmp_dir.path().join("dump"), + dump_batch_size: 16, + http_addr: "127.0.0.1:7700".to_owned(), + master_key: None, + env: "development".to_owned(), + no_analytics: true, + max_mdb_size: default_db_options.main_map_size, + max_udb_size: default_db_options.update_map_size, + http_payload_size_limit: 10000000, + ..Opt::default() + }; + + let data = Data::new(opt).unwrap(); + + Server { + uid: uid.to_string(), + data, + } + } + + pub async fn test_server() -> Self { + let mut server = Self::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + ], + "searchableAttributes": [ + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags", + ], + "displayedAttributes": [ + "id", + "isActive", + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags", + ], + }); + + server.update_all_settings(body).await; + + let dataset = include_bytes!("assets/test_set.json"); + + let body: Value = serde_json::from_slice(dataset).unwrap(); + + server.add_or_replace_multiple_documents(body).await; + server + } + + pub fn data(&self) -> &Data { + &self.data + } + + pub async fn wait_update_id(&mut self, update_id: u64) { + // try 10 times to get status, or panic to not wait forever + for _ in 0..10 { + let (response, status_code) = self.get_update_status(update_id).await; + assert_eq!(status_code, 200); + + if response["status"] == "processed" || response["status"] == "failed" { + // eprintln!("{:#?}", response); + return; + } + + delay_for(Duration::from_secs(1)).await; + } + panic!("Timeout waiting for update id"); + } + + // Global Http request GET/POST/DELETE async or sync + + pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) { + eprintln!("get_request: {}", url); + + let mut app = + test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::get().uri(url).to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + pub async fn post_request(&self, url: &str, body: Value) -> (Value, StatusCode) { + eprintln!("post_request: {}", url); + + let mut app = + test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::post() + .uri(url) + .set_json(&body) + .to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + pub async fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { + eprintln!("post_request_async: {}", url); + + let (response, status_code) = self.post_request(url, body).await; + eprintln!("response: {}", response); + assert!(response["updateId"].as_u64().is_some()); + self.wait_update_id(response["updateId"].as_u64().unwrap()) + .await; + (response, status_code) + } + + pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { + eprintln!("put_request: {}", url); + + let mut app = + test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::put() + .uri(url) + .set_json(&body) + .to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + pub async fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { + eprintln!("put_request_async: {}", url); + + let (response, status_code) = self.put_request(url, body).await; + assert!(response["updateId"].as_u64().is_some()); + assert_eq!(status_code, 202); + self.wait_update_id(response["updateId"].as_u64().unwrap()) + .await; + (response, status_code) + } + + pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) { + eprintln!("delete_request: {}", url); + + let mut app = + test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::delete().uri(url).to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + pub async fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) { + eprintln!("delete_request_async: {}", url); + + let (response, status_code) = self.delete_request(url).await; + assert!(response["updateId"].as_u64().is_some()); + assert_eq!(status_code, 202); + self.wait_update_id(response["updateId"].as_u64().unwrap()) + .await; + (response, status_code) + } + + // All Routes + + pub async fn list_indexes(&mut self) -> (Value, StatusCode) { + self.get_request("/indexes").await + } + + pub async fn create_index(&mut self, body: Value) -> (Value, StatusCode) { + self.post_request("/indexes", body).await + } + + pub async fn search_multi_index(&mut self, query: &str) -> (Value, StatusCode) { + let url = format!("/indexes/search?{}", query); + self.get_request(&url).await + } + + pub async fn get_index(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}", self.uid); + self.get_request(&url).await + } + + pub async fn update_index(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}", self.uid); + self.put_request(&url, body).await + } + + pub async fn delete_index(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}", self.uid); + self.delete_request(&url).await + } + + pub async fn search_get(&mut self, query: &str) -> (Value, StatusCode) { + let url = format!("/indexes/{}/search?{}", self.uid, query); + self.get_request(&url).await + } + + pub async fn search_post(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/search", self.uid); + self.post_request(&url, body).await + } + + pub async fn get_all_updates_status(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/updates", self.uid); + self.get_request(&url).await + } + + pub async fn get_update_status(&mut self, update_id: u64) -> (Value, StatusCode) { + let url = format!("/indexes/{}/updates/{}", self.uid, update_id); + self.get_request(&url).await + } + + pub async fn get_all_documents(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents", self.uid); + self.get_request(&url).await + } + + pub async fn add_or_replace_multiple_documents(&mut self, body: Value) { + let url = format!("/indexes/{}/documents", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn add_or_replace_multiple_documents_sync( + &mut self, + body: Value, + ) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents", self.uid); + self.post_request(&url, body).await + } + + pub async fn add_or_update_multiple_documents(&mut self, body: Value) { + let url = format!("/indexes/{}/documents", self.uid); + self.put_request_async(&url, body).await; + } + + pub async fn clear_all_documents(&mut self) { + let url = format!("/indexes/{}/documents", self.uid); + self.delete_request_async(&url).await; + } + + pub async fn get_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { + let url = format!( + "/indexes/{}/documents/{}", + self.uid, + document_id.to_string() + ); + self.get_request(&url).await + } + + pub async fn delete_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { + let url = format!( + "/indexes/{}/documents/{}", + self.uid, + document_id.to_string() + ); + self.delete_request_async(&url).await + } + + pub async fn delete_multiple_documents(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents/delete-batch", self.uid); + self.post_request_async(&url, body).await + } + + pub async fn get_all_settings(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.get_request(&url).await + } + + pub async fn update_all_settings(&mut self, body: Value) { + let url = format!("/indexes/{}/settings", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn update_all_settings_sync(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.post_request(&url, body).await + } + + pub async fn delete_all_settings(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.delete_request_async(&url).await + } + + pub async fn get_ranking_rules(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/ranking-rules", self.uid); + self.get_request(&url).await + } + + pub async fn update_ranking_rules(&mut self, body: Value) { + let url = format!("/indexes/{}/settings/ranking-rules", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn update_ranking_rules_sync(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/ranking-rules", self.uid); + self.post_request(&url, body).await + } + + pub async fn delete_ranking_rules(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/ranking-rules", self.uid); + self.delete_request_async(&url).await + } + + pub async fn get_distinct_attribute(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); + self.get_request(&url).await + } + + pub async fn update_distinct_attribute(&mut self, body: Value) { + let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn update_distinct_attribute_sync(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); + self.post_request(&url, body).await + } + + pub async fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); + self.delete_request_async(&url).await + } + + pub async fn get_primary_key(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/primary_key", self.uid); + self.get_request(&url).await + } + + pub async fn get_searchable_attributes(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); + self.get_request(&url).await + } + + pub async fn update_searchable_attributes(&mut self, body: Value) { + let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn update_searchable_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); + self.post_request(&url, body).await + } + + pub async fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); + self.delete_request_async(&url).await + } + + pub async fn get_displayed_attributes(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); + self.get_request(&url).await + } + + pub async fn update_displayed_attributes(&mut self, body: Value) { + let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn update_displayed_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); + self.post_request(&url, body).await + } + + pub async fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); + self.delete_request_async(&url).await + } + + pub async fn get_attributes_for_faceting(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); + self.get_request(&url).await + } + + pub async fn update_attributes_for_faceting(&mut self, body: Value) { + let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn update_attributes_for_faceting_sync( + &mut self, + body: Value, + ) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); + self.post_request(&url, body).await + } + + pub async fn delete_attributes_for_faceting(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); + self.delete_request_async(&url).await + } + + pub async fn get_synonyms(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/synonyms", self.uid); + self.get_request(&url).await + } + + pub async fn update_synonyms(&mut self, body: Value) { + let url = format!("/indexes/{}/settings/synonyms", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn update_synonyms_sync(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/synonyms", self.uid); + self.post_request(&url, body).await + } + + pub async fn delete_synonyms(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/synonyms", self.uid); + self.delete_request_async(&url).await + } + + pub async fn get_stop_words(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/stop-words", self.uid); + self.get_request(&url).await + } + + pub async fn update_stop_words(&mut self, body: Value) { + let url = format!("/indexes/{}/settings/stop-words", self.uid); + self.post_request_async(&url, body).await; + } + + pub async fn update_stop_words_sync(&mut self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/stop-words", self.uid); + self.post_request(&url, body).await + } + + pub async fn delete_stop_words(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/stop-words", self.uid); + self.delete_request_async(&url).await + } + + pub async fn get_index_stats(&mut self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/stats", self.uid); + self.get_request(&url).await + } + + pub async fn list_keys(&mut self) -> (Value, StatusCode) { + self.get_request("/keys").await + } + + pub async fn get_health(&mut self) -> (Value, StatusCode) { + self.get_request("/health").await + } + + pub async fn update_health(&mut self, body: Value) -> (Value, StatusCode) { + self.put_request("/health", body).await + } + + pub async fn get_version(&mut self) -> (Value, StatusCode) { + self.get_request("/version").await + } + + pub async fn get_sys_info(&mut self) -> (Value, StatusCode) { + self.get_request("/sys-info").await + } + + pub async fn get_sys_info_pretty(&mut self) -> (Value, StatusCode) { + self.get_request("/sys-info/pretty").await + } + + pub async fn trigger_dump(&self) -> (Value, StatusCode) { + self.post_request("/dumps", Value::Null).await + } + + pub async fn get_dump_status(&mut self, dump_uid: &str) -> (Value, StatusCode) { + let url = format!("/dumps/{}/status", dump_uid); + self.get_request(&url).await + } + + pub async fn trigger_dump_importation(&mut self, dump_uid: &str) -> (Value, StatusCode) { + let url = format!("/dumps/{}/import", dump_uid); + self.get_request(&url).await + } +} diff --git a/tests/dashboard.rs b/tests/dashboard.rs new file mode 100644 index 000000000..2dbaf8f7d --- /dev/null +++ b/tests/dashboard.rs @@ -0,0 +1,12 @@ +mod common; + +#[actix_rt::test] +async fn dashboard() { + let mut server = common::Server::with_uid("movies"); + + let (_response, status_code) = server.get_request("/").await; + assert_eq!(status_code, 200); + + let (_response, status_code) = server.get_request("/bulma.min.css").await; + assert_eq!(status_code, 200); +} diff --git a/tests/documents_add.rs b/tests/documents_add.rs new file mode 100644 index 000000000..382a1ed43 --- /dev/null +++ b/tests/documents_add.rs @@ -0,0 +1,222 @@ +use serde_json::json; + +mod common; + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/519 +#[actix_rt::test] +async fn check_add_documents_with_primary_key_param() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add documents + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/movies/documents?primaryKey=title"; + let (response, status_code) = server.post_request(&url, body).await; + eprintln!("{:#?}", response); + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/568 +#[actix_rt::test] +async fn check_add_documents_with_nested_boolean() { + let mut server = common::Server::with_uid("tasks"); + + // 1 - Create the index with no primary_key + + let body = json!({ "uid": "tasks" }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add a document that contains a boolean in a nested object + + let body = json!([{ + "id": 12161, + "created_at": "2019-04-10T14:57:57.522Z", + "foo": { + "bar": { + "id": 121, + "crash": false + }, + "id": 45912 + } + }]); + + let url = "/indexes/tasks/documents"; + let (response, status_code) = server.post_request(&url, body).await; + eprintln!("{:#?}", response); + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/571 +#[actix_rt::test] +async fn check_add_documents_with_nested_null() { + let mut server = common::Server::with_uid("tasks"); + + // 1 - Create the index with no primary_key + + let body = json!({ "uid": "tasks" }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add a document that contains a null in a nested object + + let body = json!([{ + "id": 0, + "foo": { + "bar": null + } + }]); + + let url = "/indexes/tasks/documents"; + let (response, status_code) = server.post_request(&url, body).await; + eprintln!("{:#?}", response); + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/574 +#[actix_rt::test] +async fn check_add_documents_with_nested_sequence() { + let mut server = common::Server::with_uid("tasks"); + + // 1 - Create the index with no primary_key + + let body = json!({ "uid": "tasks" }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add a document that contains a seq in a nested object + + let body = json!([{ + "id": 0, + "foo": { + "bar": [123,456], + "fez": [{ + "id": 255, + "baz": "leesz", + "fuzz": { + "fax": [234] + }, + "sas": [] + }], + "foz": [{ + "id": 255, + "baz": "leesz", + "fuzz": { + "fax": [234] + }, + "sas": [] + }, + { + "id": 256, + "baz": "loss", + "fuzz": { + "fax": [235] + }, + "sas": [321, 321] + }] + } + }]); + + let url = "/indexes/tasks/documents"; + let (response, status_code) = server.post_request(&url, body.clone()).await; + eprintln!("{:#?}", response); + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); + + let url = "/indexes/tasks/search?q=leesz"; + let (response, status_code) = server.get_request(&url).await; + assert_eq!(status_code, 200); + assert_eq!(response["hits"], body); +} + +#[actix_rt::test] +// test sample from #807 +async fn add_document_with_long_field() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test" })).await; + let body = json!([{ + "documentId":"de1c2adbb897effdfe0deae32a01035e46f932ce", + "rank":1, + "relurl":"/configuration/app/web.html#locations", + "section":"Web", + "site":"docs", + "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", + "title":"Locations", + "url":"/configuration/app/web.html#locations" + }]); + server.add_or_replace_multiple_documents(body).await; + let (response, _status) = server + .search_post(json!({ "q": "request_buffering" })) + .await; + assert!(!response["hits"].as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn documents_with_same_id_are_overwritten() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test"})).await; + let documents = json!([ + { + "id": 1, + "content": "test1" + }, + { + "id": 1, + "content": "test2" + }, + ]); + server.add_or_replace_multiple_documents(documents).await; + let (response, _status) = server.get_all_documents().await; + assert_eq!(response.as_array().unwrap().len(), 1); + assert_eq!( + response.as_array().unwrap()[0].as_object().unwrap()["content"], + "test2" + ); +} diff --git a/tests/documents_delete.rs b/tests/documents_delete.rs new file mode 100644 index 000000000..4353a5355 --- /dev/null +++ b/tests/documents_delete.rs @@ -0,0 +1,67 @@ +mod common; + +use serde_json::json; + +#[actix_rt::test] +async fn delete() { + let mut server = common::Server::test_server().await; + + let (_response, status_code) = server.get_document(50).await; + assert_eq!(status_code, 200); + + server.delete_document(50).await; + + let (_response, status_code) = server.get_document(50).await; + assert_eq!(status_code, 404); +} + +// Resolve the issue https://github.com/meilisearch/MeiliSearch/issues/493 +#[actix_rt::test] +async fn delete_batch() { + let mut server = common::Server::test_server().await; + + let doc_ids = vec!(50, 55, 60); + for doc_id in &doc_ids { + let (_response, status_code) = server.get_document(doc_id).await; + assert_eq!(status_code, 200); + } + + let body = serde_json::json!(&doc_ids); + server.delete_multiple_documents(body).await; + + for doc_id in &doc_ids { + let (_response, status_code) = server.get_document(doc_id).await; + assert_eq!(status_code, 404); + } +} + +#[actix_rt::test] +async fn text_clear_all_placeholder_search() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + }); + + server.create_index(body).await; + let settings = json!({ + "attributesForFaceting": ["genre"], + }); + + server.update_all_settings(settings).await; + + let documents = json!([ + { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance" }, + { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" }, + { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy" }, + { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, + { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, + { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams" } + ]); + + server.add_or_update_multiple_documents(documents).await; + server.clear_all_documents().await; + let (response, _) = server.search_post(json!({ "q": "", "facetsDistribution": ["genre"] })).await; + assert_eq!(response["nbHits"], 0); + let (response, _) = server.search_post(json!({ "q": "" })).await; + assert_eq!(response["nbHits"], 0); +} diff --git a/tests/documents_get.rs b/tests/documents_get.rs new file mode 100644 index 000000000..35e04f494 --- /dev/null +++ b/tests/documents_get.rs @@ -0,0 +1,23 @@ +use serde_json::json; +use actix_web::http::StatusCode; + +mod common; + +#[actix_rt::test] +async fn get_documents_from_unexisting_index_is_error() { + let mut server = common::Server::with_uid("test"); + let (response, status) = server.get_all_documents().await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(response["errorCode"], "index_not_found"); + assert_eq!(response["errorType"], "invalid_request_error"); + assert_eq!(response["errorLink"], "https://docs.meilisearch.com/errors#index_not_found"); +} + +#[actix_rt::test] +async fn get_empty_documents_list() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test" })).await; + let (response, status) = server.get_all_documents().await; + assert_eq!(status, StatusCode::OK); + assert!(response.as_array().unwrap().is_empty()); +} diff --git a/tests/dump.rs b/tests/dump.rs new file mode 100644 index 000000000..701b754aa --- /dev/null +++ b/tests/dump.rs @@ -0,0 +1,395 @@ +use assert_json_diff::{assert_json_eq, assert_json_include}; +use meilisearch_http::helpers::compression; +use serde_json::{json, Value}; +use std::fs::File; +use std::path::Path; +use std::thread; +use std::time::Duration; +use tempfile::TempDir; + +#[macro_use] mod common; + +async fn trigger_and_wait_dump(server: &mut common::Server) -> String { + let (value, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + for _ in 0..20 as u8 { + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + assert_eq!(status_code, 200); + assert_ne!(value["status"].as_str(), Some("dump_process_failed")); + + if value["status"].as_str() == Some("done") { return dump_uid } + thread::sleep(Duration::from_millis(100)); + } + + unreachable!("dump creation runned out of time") +} + +fn current_db_version() -> (String, String, String) { + let current_version_major = env!("CARGO_PKG_VERSION_MAJOR").to_string(); + let current_version_minor = env!("CARGO_PKG_VERSION_MINOR").to_string(); + let current_version_patch = env!("CARGO_PKG_VERSION_PATCH").to_string(); + + (current_version_major, current_version_minor, current_version_patch) +} + +fn current_dump_version() -> String { + "V1".into() +} + +fn read_all_jsonline(r: R) -> Value { + let deserializer = serde_json::Deserializer::from_reader(r); + let iterator = deserializer.into_iter::(); + + json!(iterator.map(|v| v.unwrap()).collect::>()) +} + +#[actix_rt::test] +#[ignore] +async fn trigger_dump_should_return_ok() { + let server = common::Server::test_server().await; + + let (_, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); +} + +#[actix_rt::test] +#[ignore] +async fn trigger_dump_twice_should_return_conflict() { + let server = common::Server::test_server().await; + + let expected = json!({ + "message": "Another dump is already in progress", + "errorCode": "dump_already_in_progress", + "errorType": "invalid_request_error", + "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" + }); + + let (_, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); + + let (value, status_code) = server.trigger_dump().await; + + + assert_json_eq!(expected, value, ordered: false); + assert_eq!(status_code, 409); +} + +#[actix_rt::test] +#[ignore] +async fn trigger_dump_concurently_should_return_conflict() { + let server = common::Server::test_server().await; + + let expected = json!({ + "message": "Another dump is already in progress", + "errorCode": "dump_already_in_progress", + "errorType": "invalid_request_error", + "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" + }); + + let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump()); + + assert_json_eq!(expected, value_2, ordered: false); + assert_eq!(status_code_2, 409); +} + +#[actix_rt::test] +#[ignore] +async fn get_dump_status_early_should_return_in_progress() { + let mut server = common::Server::test_server().await; + + + + let (value, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + let expected = json!({ + "uid": dump_uid, + "status": "in_progress" + }); + + assert_eq!(status_code, 200); + + assert_json_eq!(expected, value, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn get_dump_status_should_return_done() { + let mut server = common::Server::test_server().await; + + + let (value, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + let expected = json!({ + "uid": dump_uid.clone(), + "status": "done" + }); + + thread::sleep(Duration::from_secs(1)); // wait dump until process end + + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + assert_eq!(status_code, 200); + + assert_json_eq!(expected, value, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn get_dump_status_should_return_error_provoking_it() { + let mut server = common::Server::test_server().await; + + + let (value, status_code) = server.trigger_dump().await; + + // removing destination directory provoking `No such file or directory` error + std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap(); + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + let expected = json!({ + "uid": dump_uid.clone(), + "status": "failed", + "message": "Dump process failed: compressing dump; No such file or directory (os error 2)", + "errorCode": "dump_process_failed", + "errorType": "internal_error", + "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed" + }); + + thread::sleep(Duration::from_secs(1)); // wait dump until process end + + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + assert_eq!(status_code, 200); + + assert_json_eq!(expected, value, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn dump_metadata_should_be_valid() { + let mut server = common::Server::test_server().await; + + let body = json!({ + "uid": "test2", + "primaryKey": "test2_id", + }); + + server.create_index(body).await; + + let uid = trigger_and_wait_dump(&mut server).await; + + let dumps_dir = Path::new(&server.data().dumps_dir); + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path(); + + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); + + let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); + let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); + + // fields are randomly ordered + metadata.get_mut("indexes").unwrap() + .as_array_mut().unwrap() + .sort_by(|a, b| + a.get("uid").unwrap().as_str().cmp(&b.get("uid").unwrap().as_str()) + ); + + let (major, minor, patch) = current_db_version(); + + let expected = json!({ + "indexes": [{ + "uid": "test", + "primaryKey": "id", + }, { + "uid": "test2", + "primaryKey": "test2_id", + } + ], + "dbVersion": format!("{}.{}.{}", major, minor, patch), + "dumpVersion": current_dump_version() + }); + + assert_json_include!(expected: expected, actual: metadata); +} + +#[actix_rt::test] +#[ignore] +async fn dump_gzip_should_have_been_created() { + let mut server = common::Server::test_server().await; + + + let dump_uid = trigger_and_wait_dump(&mut server).await; + let dumps_dir = Path::new(&server.data().dumps_dir); + + let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid)); + assert!(File::open(compressed_path).is_ok()); +} + +#[actix_rt::test] +#[ignore] +async fn dump_index_settings_should_be_valid() { + let mut server = common::Server::test_server().await; + + let expected = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": "email", + "searchableAttributes": [ + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "displayedAttributes": [ + "id", + "isActive", + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "stopWords": [ + "in", + "ad" + ], + "synonyms": { + "wolverine": ["xmen", "logan"], + "logan": ["wolverine", "xmen"] + }, + "attributesForFaceting": [ + "gender", + "color", + "tags" + ] + }); + + server.update_all_settings(expected.clone()).await; + + let uid = trigger_and_wait_dump(&mut server).await; + + let dumps_dir = Path::new(&server.data().dumps_dir); + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path(); + + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); + + let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); + let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); + + assert_json_eq!(expected, settings, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn dump_index_documents_should_be_valid() { + let mut server = common::Server::test_server().await; + + let dataset = include_bytes!("assets/dumps/v1/test/documents.jsonl"); + let mut slice: &[u8] = dataset; + + let expected: Value = read_all_jsonline(&mut slice); + + let uid = trigger_and_wait_dump(&mut server).await; + + let dumps_dir = Path::new(&server.data().dumps_dir); + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path(); + + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); + + let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); + let documents = read_all_jsonline(file); + + assert_json_eq!(expected, documents, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn dump_index_updates_should_be_valid() { + let mut server = common::Server::test_server().await; + + let dataset = include_bytes!("assets/dumps/v1/test/updates.jsonl"); + let mut slice: &[u8] = dataset; + + let expected: Value = read_all_jsonline(&mut slice); + + let uid = trigger_and_wait_dump(&mut server).await; + + let dumps_dir = Path::new(&server.data().dumps_dir); + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path(); + + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); + + let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); + let mut updates = read_all_jsonline(file); + + + // hotfix until #943 is fixed (https://github.com/meilisearch/MeiliSearch/issues/943) + updates.as_array_mut().unwrap() + .get_mut(0).unwrap() + .get_mut("type").unwrap() + .get_mut("settings").unwrap() + .get_mut("displayed_attributes").unwrap() + .get_mut("Update").unwrap() + .as_array_mut().unwrap().sort_by(|a, b| a.as_str().cmp(&b.as_str())); + + eprintln!("{}\n", updates.to_string()); + eprintln!("{}", expected.to_string()); + assert_json_include!(expected: expected, actual: updates); +} + +#[actix_rt::test] +#[ignore] +async fn get_unexisting_dump_status_should_return_not_found() { + let mut server = common::Server::test_server().await; + + let (_, status_code) = server.get_dump_status("4242").await; + + assert_eq!(status_code, 404); +} diff --git a/tests/errors.rs b/tests/errors.rs new file mode 100644 index 000000000..e11483356 --- /dev/null +++ b/tests/errors.rs @@ -0,0 +1,200 @@ +mod common; + +use std::thread; +use std::time::Duration; + +use actix_http::http::StatusCode; +use serde_json::{json, Map, Value}; + +macro_rules! assert_error { + ($code:literal, $type:literal, $status:path, $req:expr) => { + let (response, status_code) = $req; + assert_eq!(status_code, $status); + assert_eq!(response["errorCode"].as_str().unwrap(), $code); + assert_eq!(response["errorType"].as_str().unwrap(), $type); + }; +} + +macro_rules! assert_error_async { + ($code:literal, $type:literal, $server:expr, $req:expr) => { + let (response, _) = $req; + let update_id = response["updateId"].as_u64().unwrap(); + for _ in 1..10 { + let (response, status_code) = $server.get_update_status(update_id).await; + assert_eq!(status_code, StatusCode::OK); + if response["status"] == "processed" || response["status"] == "failed" { + println!("response: {}", response); + assert_eq!(response["status"], "failed"); + assert_eq!(response["errorCode"], $code); + assert_eq!(response["errorType"], $type); + return + } + thread::sleep(Duration::from_secs(1)); + } + }; +} + +#[actix_rt::test] +async fn index_already_exists_error() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test" + }); + let (response, status_code) = server.create_index(body.clone()).await; + println!("{}", response); + assert_eq!(status_code, StatusCode::CREATED); + + let (response, status_code) = server.create_index(body.clone()).await; + println!("{}", response); + + assert_error!( + "index_already_exists", + "invalid_request_error", + StatusCode::BAD_REQUEST, + (response, status_code)); +} + +#[actix_rt::test] +async fn index_not_found_error() { + let mut server = common::Server::with_uid("test"); + assert_error!( + "index_not_found", + "invalid_request_error", + StatusCode::NOT_FOUND, + server.get_index().await); +} + +#[actix_rt::test] +async fn primary_key_already_present_error() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "test" + }); + server.create_index(body.clone()).await; + let body = json!({ + "primaryKey": "t" + }); + assert_error!( + "primary_key_already_present", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.update_index(body).await); +} + +#[actix_rt::test] +async fn max_field_limit_exceeded_error() { + let mut server = common::Server::test_server().await; + let body = json!({ + "uid": "test", + }); + server.create_index(body).await; + let mut doc = Map::with_capacity(70_000); + doc.insert("id".into(), Value::String("foo".into())); + for i in 0..69_999 { + doc.insert(format!("field{}", i), Value::String("foo".into())); + } + let docs = json!([doc]); + assert_error_async!( + "max_fields_limit_exceeded", + "invalid_request_error", + server, + server.add_or_replace_multiple_documents_sync(docs).await); +} + +#[actix_rt::test] +async fn missing_document_id() { + let mut server = common::Server::test_server().await; + let body = json!({ + "uid": "test", + "primaryKey": "test" + }); + server.create_index(body).await; + let docs = json!([ + { + "foo": "bar", + } + ]); + assert_error_async!( + "missing_document_id", + "invalid_request_error", + server, + server.add_or_replace_multiple_documents_sync(docs).await); +} + +#[actix_rt::test] +async fn facet_error() { + let mut server = common::Server::test_server().await; + let search = json!({ + "q": "foo", + "facetFilters": ["test:hello"] + }); + assert_error!( + "invalid_facet", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.search_post(search).await); +} + +#[actix_rt::test] +async fn filters_error() { + let mut server = common::Server::test_server().await; + let search = json!({ + "q": "foo", + "filters": "fo:12" + }); + assert_error!( + "invalid_filter", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.search_post(search).await); +} + +#[actix_rt::test] +async fn bad_request_error() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "foo": "bar", + }); + assert_error!( + "bad_request", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.search_post(body).await); +} + +#[actix_rt::test] +async fn document_not_found_error() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({"uid": "test"})).await; + assert_error!( + "document_not_found", + "invalid_request_error", + StatusCode::NOT_FOUND, + server.get_document(100).await); +} + +#[actix_rt::test] +async fn payload_too_large_error() { + let mut server = common::Server::with_uid("test"); + let bigvec = vec![0u64; 10_000_000]; // 80mb + assert_error!( + "payload_too_large", + "invalid_request_error", + StatusCode::PAYLOAD_TOO_LARGE, + server.create_index(json!(bigvec)).await); +} + +#[actix_rt::test] +async fn missing_primary_key_error() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({"uid": "test"})).await; + let document = json!([{ + "content": "test" + }]); + assert_error!( + "missing_primary_key", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.add_or_replace_multiple_documents_sync(document).await); +} diff --git a/tests/health.rs b/tests/health.rs new file mode 100644 index 000000000..f72127431 --- /dev/null +++ b/tests/health.rs @@ -0,0 +1,11 @@ +mod common; + +#[actix_rt::test] +async fn test_healthyness() { + let mut server = common::Server::with_uid("movies"); + + // Check that the server is healthy + + let (_response, status_code) = server.get_health().await; + assert_eq!(status_code, 204); +} diff --git a/tests/index.rs b/tests/index.rs new file mode 100644 index 000000000..271507e03 --- /dev/null +++ b/tests/index.rs @@ -0,0 +1,809 @@ +use actix_web::http::StatusCode; +use assert_json_diff::assert_json_eq; +use serde_json::{json, Value}; + +mod common; + +#[actix_rt::test] +async fn create_index_with_name() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "name": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 8); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid.len(), r1_uid.len()); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); +} + +#[actix_rt::test] +async fn create_index_with_uid() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "uid": "movies", + }); + + let (res1_value, status_code) = server.create_index(body.clone()).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid, "movies"); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 1.5 verify that error is thrown when trying to create the same index + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + assert_eq!( + response["errorCode"].as_str().unwrap(), + "index_already_exists" + ); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid, r1_uid); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); +} + +#[actix_rt::test] +async fn create_index_with_name_and_uid() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "name": "Films", + "uid": "fr_movies", + }); + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "Films"); + assert_eq!(r1_uid, "fr_movies"); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid, r1_uid); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); +} + +#[actix_rt::test] +async fn rename_index() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "name": "movies", + "uid": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 6); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Update an index name + + let body = json!({ + "name": "TV Shows", + }); + + let (res2_value, status_code) = server.update_index(body).await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_object().unwrap().len(), 5); + let r2_name = res2_value["name"].as_str().unwrap(); + let r2_uid = res2_value["uid"].as_str().unwrap(); + let r2_created_at = res2_value["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, "TV Shows"); + assert_eq!(r2_uid, r1_uid); + assert_eq!(r2_created_at, r1_created_at); + assert!(r2_updated_at.len() > 1); + + // 3 - Check the list of indexes + + let (res3_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res3_value.as_array().unwrap().len(), 1); + assert_eq!(res3_value[0].as_object().unwrap().len(), 5); + let r3_name = res3_value[0]["name"].as_str().unwrap(); + let r3_uid = res3_value[0]["uid"].as_str().unwrap(); + let r3_created_at = res3_value[0]["createdAt"].as_str().unwrap(); + let r3_updated_at = res3_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r3_name, r2_name); + assert_eq!(r3_uid.len(), r1_uid.len()); + assert_eq!(r3_created_at.len(), r1_created_at.len()); + assert_eq!(r3_updated_at.len(), r2_updated_at.len()); +} + +#[actix_rt::test] +async fn delete_index_and_recreate_it() { + let mut server = common::Server::with_uid("movies"); + + // 0 - delete unexisting index is error + + let (response, status_code) = server.delete_request("/indexes/test").await; + assert_eq!(status_code, 404); + assert_eq!(&response["errorCode"], "index_not_found"); + + // 1 - Create a new index + + let body = json!({ + "name": "movies", + "uid": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 6); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid.len(), r1_uid.len()); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); + + // 3- Delete an index + + let (_res2_value, status_code) = server.delete_index().await; + + assert_eq!(status_code, 204); + + // 4 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 0); + + // 5 - Create a new index + + let body = json!({ + "name": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 8); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 6 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid.len(), r1_uid.len()); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); +} + +#[actix_rt::test] +async fn check_multiples_indexes() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "name": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 8); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_0_name = res2_value[0]["name"].as_str().unwrap(); + let r2_0_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_0_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_0_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_0_name, r1_name); + assert_eq!(r2_0_uid.len(), r1_uid.len()); + assert_eq!(r2_0_created_at.len(), r1_created_at.len()); + assert_eq!(r2_0_updated_at.len(), r1_updated_at.len()); + + // 3 - Create a new index + + let body = json!({ + "name": "films", + }); + + let (res3_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res3_value.as_object().unwrap().len(), 5); + let r3_name = res3_value["name"].as_str().unwrap(); + let r3_uid = res3_value["uid"].as_str().unwrap(); + let r3_created_at = res3_value["createdAt"].as_str().unwrap(); + let r3_updated_at = res3_value["updatedAt"].as_str().unwrap(); + assert_eq!(r3_name, "films"); + assert_eq!(r3_uid.len(), 8); + assert!(r3_created_at.len() > 1); + assert!(r3_updated_at.len() > 1); + + // 4 - Check the list of indexes + + let (res4_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res4_value.as_array().unwrap().len(), 2); + assert_eq!(res4_value[0].as_object().unwrap().len(), 5); + let r4_0_name = res4_value[0]["name"].as_str().unwrap(); + let r4_0_uid = res4_value[0]["uid"].as_str().unwrap(); + let r4_0_created_at = res4_value[0]["createdAt"].as_str().unwrap(); + let r4_0_updated_at = res4_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(res4_value[1].as_object().unwrap().len(), 5); + let r4_1_name = res4_value[1]["name"].as_str().unwrap(); + let r4_1_uid = res4_value[1]["uid"].as_str().unwrap(); + let r4_1_created_at = res4_value[1]["createdAt"].as_str().unwrap(); + let r4_1_updated_at = res4_value[1]["updatedAt"].as_str().unwrap(); + if r4_0_name == r1_name { + assert_eq!(r4_0_name, r1_name); + assert_eq!(r4_0_uid.len(), r1_uid.len()); + assert_eq!(r4_0_created_at.len(), r1_created_at.len()); + assert_eq!(r4_0_updated_at.len(), r1_updated_at.len()); + } else { + assert_eq!(r4_0_name, r3_name); + assert_eq!(r4_0_uid.len(), r3_uid.len()); + assert_eq!(r4_0_created_at.len(), r3_created_at.len()); + assert_eq!(r4_0_updated_at.len(), r3_updated_at.len()); + } + if r4_1_name == r1_name { + assert_eq!(r4_1_name, r1_name); + assert_eq!(r4_1_uid.len(), r1_uid.len()); + assert_eq!(r4_1_created_at.len(), r1_created_at.len()); + assert_eq!(r4_1_updated_at.len(), r1_updated_at.len()); + } else { + assert_eq!(r4_1_name, r3_name); + assert_eq!(r4_1_uid.len(), r3_uid.len()); + assert_eq!(r4_1_created_at.len(), r3_created_at.len()); + assert_eq!(r4_1_updated_at.len(), r3_updated_at.len()); + } +} + +#[actix_rt::test] +async fn create_index_failed() { + let mut server = common::Server::with_uid("movies"); + + // 2 - Push index creation with empty json body + + let body = json!({}); + + let (res_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = res_value["message"].as_str().unwrap(); + assert_eq!(res_value.as_object().unwrap().len(), 4); + assert_eq!(message, "Index creation must have an uid"); + + // 3 - Create a index with extra data + + let body = json!({ + "name": "movies", + "active": true + }); + + let (_res_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + + // 3 - Create a index with wrong data type + + let body = json!({ + "name": "movies", + "uid": 0 + }); + + let (_res_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); +} + +// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/492 +#[actix_rt::test] +async fn create_index_with_primary_key_and_index() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index + + let body = json!({ + "uid": "movies", + "primaryKey": "id", + }); + + let (_response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + + // 2 - Add content + + let body = json!([{ + "id": 123, + "text": "The mask" + }]); + + server.add_or_replace_multiple_documents(body.clone()).await; + + // 3 - Retreive document + + let (response, _status_code) = server.get_document(123).await; + + let expect = json!({ + "id": 123, + "text": "The mask" + }); + + assert_json_eq!(response, expect, ordered: false); +} + +// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/497 +// Test when the given index uid is not valid +// Should have a 400 status code +// Should have the right error message +#[actix_rt::test] +async fn create_index_with_invalid_uid() { + let mut server = common::Server::with_uid(""); + + // 1 - Create the index with invalid uid + + let body = json!({ + "uid": "the movies" + }); + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = response["message"].as_str().unwrap(); + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); + + // 2 - Create the index with invalid uid + + let body = json!({ + "uid": "%$#" + }); + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = response["message"].as_str().unwrap(); + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); + + // 3 - Create the index with invalid uid + + let body = json!({ + "uid": "the~movies" + }); + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = response["message"].as_str().unwrap(); + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); + + // 4 - Create the index with invalid uid + + let body = json!({ + "uid": "🎉" + }); + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = response["message"].as_str().unwrap(); + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); +} + +// Test that it's possible to add primary_key if it's not already set on index creation +#[actix_rt::test] +async fn create_index_and_add_indentifier_after() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Update the index and add an primary_key. + + let body = json!({ + "primaryKey": "id", + }); + + let (response, status_code) = server.update_index(body).await; + assert_eq!(status_code, 200); + eprintln!("response: {:#?}", response); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); + + // 3 - Get index to verify if the primary_key is good + + let (response, status_code) = server.get_index().await; + assert_eq!(status_code, 200); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); +} + +// Test that it's impossible to change the primary_key +#[actix_rt::test] +async fn create_index_and_update_indentifier_after() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + "primaryKey": "id", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); + + // 2 - Update the index and add an primary_key. + + let body = json!({ + "primaryKey": "skuid", + }); + + let (_response, status_code) = server.update_index(body).await; + assert_eq!(status_code, 400); + + // 3 - Get index to verify if the primary_key still the first one + + let (response, status_code) = server.get_index().await; + assert_eq!(status_code, 200); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); +} + +// Test that schema inference work well +#[actix_rt::test] +async fn create_index_without_primary_key_and_add_document() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add a document + + let body = json!([{ + "id": 123, + "title": "I'm a legend", + }]); + + server.add_or_update_multiple_documents(body).await; + + // 3 - Get index to verify if the primary_key is good + + let (response, status_code) = server.get_index().await; + assert_eq!(status_code, 200); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); +} + +// Test search with no primary_key +#[actix_rt::test] +async fn create_index_without_primary_key_and_search() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Search + + let query = "q=captain&limit=3"; + + let (response, status_code) = server.search_get(&query).await; + assert_eq!(status_code, 200); + assert_eq!(response["hits"].as_array().unwrap().len(), 0); +} + +// Test the error message when we push an document update and impossibility to find primary key +// Test issue https://github.com/meilisearch/MeiliSearch/issues/517 +#[actix_rt::test] +async fn check_add_documents_without_primary_key() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2- Add document + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let (response, status_code) = server.add_or_replace_multiple_documents_sync(body).await; + + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(response["errorCode"], "missing_primary_key"); + assert_eq!(status_code, 400); +} + +#[actix_rt::test] +async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { + let mut server = common::Server::with_uid("movies"); + + let body = json!({ + "uid": "movies", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + let dataset = include_bytes!("./assets/test_set.json"); + + let body: Value = serde_json::from_slice(dataset).unwrap(); + + // 2. Index the documents from movies.json, present inside of assets directory + server.add_or_replace_multiple_documents(body).await; + + // 3. Fetch the status of the indexing done above. + let (response, status_code) = server.get_all_updates_status().await; + + // 4. Verify the fetch is successful and indexing status is 'processed' + assert_eq!(status_code, 200); + assert_eq!(response[0]["status"], "processed"); +} + +#[actix_rt::test] +async fn get_empty_index() { + let mut server = common::Server::with_uid("test"); + let (response, _status) = server.list_indexes().await; + assert!(response.as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn create_and_list_multiple_indices() { + let mut server = common::Server::with_uid("test"); + for i in 0..10 { + server + .create_index(json!({ "uid": format!("test{}", i) })) + .await; + } + let (response, _status) = server.list_indexes().await; + assert_eq!(response.as_array().unwrap().len(), 10); +} + +#[actix_rt::test] +async fn get_unexisting_index_is_error() { + let mut server = common::Server::with_uid("test"); + let (response, status) = server.get_index().await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(response["errorCode"], "index_not_found"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn create_index_twice_is_error() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test" })).await; + let (response, status) = server.create_index(json!({ "uid": "test" })).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["errorCode"], "index_already_exists"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn badly_formatted_index_name_is_error() { + let mut server = common::Server::with_uid("$__test"); + let (response, status) = server.create_index(json!({ "uid": "$__test" })).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["errorCode"], "invalid_index_uid"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn correct_response_no_primary_key_index() { + let mut server = common::Server::with_uid("test"); + let (response, _status) = server.create_index(json!({ "uid": "test" })).await; + assert_eq!(response["primaryKey"], Value::Null); +} + +#[actix_rt::test] +async fn correct_response_with_primary_key_index() { + let mut server = common::Server::with_uid("test"); + let (response, _status) = server + .create_index(json!({ "uid": "test", "primaryKey": "test" })) + .await; + assert_eq!(response["primaryKey"], "test"); +} + +#[actix_rt::test] +async fn udpate_unexisting_index_is_error() { + let mut server = common::Server::with_uid("test"); + let (response, status) = server.update_index(json!({ "primaryKey": "foobar" })).await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(response["errorCode"], "index_not_found"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn update_existing_primary_key_is_error() { + let mut server = common::Server::with_uid("test"); + server + .create_index(json!({ "uid": "test", "primaryKey": "key" })) + .await; + let (response, status) = server.update_index(json!({ "primaryKey": "test2" })).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["errorCode"], "primary_key_already_present"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn test_facets_distribution_attribute() { + let mut server = common::Server::test_server().await; + + let (response, _status_code) = server.get_index_stats().await; + + let expected = json!({ + "isIndexing": false, + "numberOfDocuments":77, + "fieldsDistribution":{ + "age":77, + "gender":77, + "phone":77, + "name":77, + "registered":77, + "latitude":77, + "email":77, + "tags":77, + "longitude":77, + "color":77, + "address":77, + "balance":77, + "about":77, + "picture":77, + }, + }); + + assert_json_eq!(expected, response, ordered: true); +} diff --git a/tests/index_update.rs b/tests/index_update.rs new file mode 100644 index 000000000..df4639252 --- /dev/null +++ b/tests/index_update.rs @@ -0,0 +1,200 @@ +use serde_json::json; +use serde_json::Value; +use assert_json_diff::assert_json_include; + +mod common; + +#[actix_rt::test] +async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + let dataset = include_bytes!("assets/test_set.json"); + + let body: Value = serde_json::from_slice(dataset).unwrap(); + + // 2. Index the documents from movies.json, present inside of assets directory + server.add_or_replace_multiple_documents(body).await; + + // 3. Fetch the status of the indexing done above. + let (response, status_code) = server.get_all_updates_status().await; + + // 4. Verify the fetch is successful and indexing status is 'processed' + assert_eq!(status_code, 200); + assert_eq!(response[0]["status"], "processed"); +} + +#[actix_rt::test] +async fn return_error_when_get_update_status_of_unexisting_index() { + let mut server = common::Server::with_uid("test"); + + // 1. Fetch the status of unexisting index. + let (_, status_code) = server.get_all_updates_status().await; + + // 2. Verify the fetch returned 404 + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn return_empty_when_get_update_status_of_empty_index() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2. Fetch the status of empty index. + let (response, status_code) = server.get_all_updates_status().await; + + // 3. Verify the fetch is successful, and no document are returned + assert_eq!(status_code, 200); + assert_eq!(response, json!([])); +} + +#[actix_rt::test] +async fn return_update_status_of_pushed_documents() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + + let bodies = vec![ + json!([{ + "title": "Test", + "comment": "comment test" + }]), + json!([{ + "title": "Test1", + "comment": "comment test1" + }]), + json!([{ + "title": "Test2", + "comment": "comment test2" + }]), + ]; + + let mut update_ids = Vec::new(); + + let url = "/indexes/test/documents?primaryKey=title"; + for body in bodies { + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + update_ids.push(update_id); + } + + // 2. Fetch the status of index. + let (response, status_code) = server.get_all_updates_status().await; + + // 3. Verify the fetch is successful, and updates are returned + + let expected = json!([{ + "type": { + "name": "DocumentsAddition", + "number": 1, + }, + "updateId": update_ids[0] + },{ + "type": { + "name": "DocumentsAddition", + "number": 1, + }, + "updateId": update_ids[1] + },{ + "type": { + "name": "DocumentsAddition", + "number": 1, + }, + "updateId": update_ids[2] + },]); + + assert_eq!(status_code, 200); + assert_json_include!(actual: json!(response), expected: expected); +} + +#[actix_rt::test] +async fn return_error_if_index_does_not_exist() { + let mut server = common::Server::with_uid("test"); + + let (response, status_code) = server.get_update_status(42).await; + + assert_eq!(status_code, 404); + assert_eq!(response["errorCode"], "index_not_found"); +} + +#[actix_rt::test] +async fn return_error_if_update_does_not_exist() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + let (response, status_code) = server.get_update_status(42).await; + + assert_eq!(status_code, 404); + assert_eq!(response["errorCode"], "not_found"); +} + +#[actix_rt::test] +async fn should_return_existing_update() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/test/documents?primaryKey=title"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 202); + + let update_id = response["updateId"].as_u64().unwrap(); + + let expected = json!({ + "type": { + "name": "DocumentsAddition", + "number": 1, + }, + "updateId": update_id + }); + + let (response, status_code) = server.get_update_status(update_id).await; + + assert_eq!(status_code, 200); + assert_json_include!(actual: json!(response), expected: expected); +} diff --git a/tests/lazy_index_creation.rs b/tests/lazy_index_creation.rs new file mode 100644 index 000000000..6730db82e --- /dev/null +++ b/tests/lazy_index_creation.rs @@ -0,0 +1,446 @@ +use serde_json::json; + +mod common; + +#[actix_rt::test] +async fn create_index_lazy_by_pushing_documents() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Add documents + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/movies/documents?primaryKey=title"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +#[actix_rt::test] +async fn create_index_lazy_by_pushing_documents_and_discover_pk() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Add documents + + let body = json!([{ + "id": 1, + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/movies/documents"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +#[actix_rt::test] +async fn create_index_lazy_by_pushing_documents_with_wrong_name() { + let server = common::Server::with_uid("wrong&name"); + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/wrong&name/documents?primaryKey=title"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_index_uid"); +} + +#[actix_rt::test] +async fn create_index_lazy_add_documents_failed() { + let mut server = common::Server::with_uid("wrong&name"); + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/wrong&name/documents"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_index_uid"); + + let (_, status_code) = server.get_index().await; + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_settings() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["name"], + }); + + server.update_all_settings(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_settings_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "other", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "anotherSettings": ["name"], + }); + + let (_, status_code) = server.update_all_settings_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_ranking_rules() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ]); + + server.update_ranking_rules(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_ranking_rules_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!({ + "rankingRules": 123, + }); + + let (_, status_code) = server.update_ranking_rules_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_distinct_attribute() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!("type"); + + server.update_distinct_attribute(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_distinct_attribute_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (resp, status_code) = server.update_distinct_attribute_sync(body.clone()).await; + eprintln!("resp: {:?}", resp); + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (resp, status_code) = server.get_all_settings().await; + eprintln!("resp: {:?}", resp); + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_searchable_attributes() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(["title", "description"]); + + server.update_searchable_attributes(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_searchable_attributes_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server.update_searchable_attributes_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_displayed_attributes() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(["title", "description"]); + + server.update_displayed_attributes(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_displayed_attributes_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server.update_displayed_attributes_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_attributes_for_faceting() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(["title", "description"]); + + server.update_attributes_for_faceting(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_attributes_for_faceting_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server + .update_attributes_for_faceting_sync(body.clone()) + .await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_synonyms() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!({ + "road": ["street", "avenue"], + "street": ["avenue"], + }); + + server.update_synonyms(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_synonyms_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server.update_synonyms_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_stop_words() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(["le", "la", "les"]); + + server.update_stop_words(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_stop_words_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server.update_stop_words_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} diff --git a/tests/placeholder_search.rs b/tests/placeholder_search.rs new file mode 100644 index 000000000..048ab7f8b --- /dev/null +++ b/tests/placeholder_search.rs @@ -0,0 +1,629 @@ +use std::convert::Into; + +use serde_json::json; +use serde_json::Value; +use std::cell::RefCell; +use std::sync::Mutex; + +#[macro_use] +mod common; + +#[actix_rt::test] +async fn placeholder_search_with_limit() { + let mut server = common::Server::test_server().await; + + let query = json! ({ + "limit": 3 + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + assert_eq!(response["hits"].as_array().unwrap().len(), 3); + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_offset() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "limit": 6, + }); + + // hack to take a value out of macro (must implement UnwindSafe) + let expected = Mutex::new(RefCell::new(Vec::new())); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + // take results at offset 3 as reference + let lock = expected.lock().unwrap(); + lock.replace(response["hits"].as_array().unwrap()[3..6].to_vec()); + }); + let expected = expected.into_inner().unwrap().into_inner(); + + let query = json!({ + "limit": 3, + "offset": 3, + }); + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + let response = response["hits"].as_array().unwrap(); + assert_eq!(&expected, response); + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_attribute_to_highlight_wildcard() { + // there should be no highlight in placeholder search + let mut server = common::Server::test_server().await; + + let query = json!({ + "limit": 1, + "attributesToHighlight": ["*"] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + let result = response["hits"].as_array().unwrap()[0].as_object().unwrap(); + for value in result.values() { + assert!(value.to_string().find("").is_none()); + } + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_matches() { + // matches is always empty + let mut server = common::Server::test_server().await; + + let query = json!({ + "matches": true + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + let result = response["hits"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_object().unwrap()["_matchesInfo"].clone()) + .all(|m| m.as_object().unwrap().is_empty()); + assert!(result); + }); +} + +#[actix_rt::test] +async fn placeholder_search_witch_crop() { + // placeholder search crop always crop from beggining + let mut server = common::Server::test_server().await; + + let query = json!({ + "attributesToCrop": ["about"], + "cropLength": 20 + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + + let hits = response["hits"].as_array().unwrap(); + + for hit in hits { + let hit = hit.as_object().unwrap(); + let formatted = hit["_formatted"].as_object().unwrap(); + + let about = hit["about"].as_str().unwrap(); + let about_formatted = formatted["about"].as_str().unwrap(); + // the formatted about length should be about 20 characters long + assert!(about_formatted.len() < 20 + 10); + // the formatted part should be located at the beginning of the original one + assert_eq!(about.find(&about_formatted).unwrap(), 0); + } + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_attributes_to_retrieve() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "limit": 1, + "attributesToRetrieve": ["gender", "about"], + }); + + test_post_get_search!(server, query, |response, _status_code| { + let hit = response["hits"].as_array().unwrap()[0].as_object().unwrap(); + assert_eq!(hit.values().count(), 2); + let _ = hit["gender"]; + let _ = hit["about"]; + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_filter() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "filters": "color='green'" + }); + + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green")); + }); + + let query = json!({ + "filters": "tags=bug" + }); + + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + let value = Value::String(String::from("bug")); + assert!(hits + .iter() + .all(|v| v["tags"].as_array().unwrap().contains(&value))); + }); + + let query = json!({ + "filters": "color='green' AND (tags='bug' OR tags='wontfix')" + }); + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + let bug = Value::String(String::from("bug")); + let wontfix = Value::String(String::from("wontfix")); + assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green" + && v["tags"].as_array().unwrap().contains(&bug) + || v["tags"].as_array().unwrap().contains(&wontfix))); + }); +} + +#[actix_rt::test] +async fn placeholder_test_faceted_search_valid() { + let mut server = common::Server::test_server().await; + + // simple tests on attributes with string value + let body = json!({ + "attributesForFaceting": ["color"] + }); + + server.update_all_settings(body).await; + + let query = json!({ + "facetFilters": ["color:green"] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "Green")); + }); + + let query = json!({ + "facetFilters": [["color:blue"]] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue")); + }); + + let query = json!({ + "facetFilters": ["color:Blue"] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue")); + }); + + // test on arrays: ["tags:bug"] + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + + server.update_all_settings(body).await; + + let query = json!({ + "facetFilters": ["tags:bug"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value + .get("tags") + .unwrap() + .as_array() + .unwrap() + .contains(&Value::String("bug".to_owned())))); + }); + + // test and: ["color:blue", "tags:bug"] + let query = json!({ + "facetFilters": ["color:blue", "tags:bug"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue" + && value + .get("tags") + .unwrap() + .as_array() + .unwrap() + .contains(&Value::String("bug".to_owned())))); + }); + + // test or: [["color:blue", "color:green"]] + let query = json!({ + "facetFilters": [["color:blue", "color:green"]] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue" + || value.get("color").unwrap() == "Green")); + }); + // test and-or: ["tags:bug", ["color:blue", "color:green"]] + let query = json!({ + "facetFilters": ["tags:bug", ["color:blue", "color:green"]] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value + .get("tags") + .unwrap() + .as_array() + .unwrap() + .contains(&Value::String("bug".to_owned())) + && (value.get("color").unwrap() == "blue" + || value.get("color").unwrap() == "Green"))); + }); +} + +#[actix_rt::test] +async fn placeholder_test_faceted_search_invalid() { + let mut server = common::Server::test_server().await; + + //no faceted attributes set + let query = json!({ + "facetFilters": ["color:blue"] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + server.update_all_settings(body).await; + // empty arrays are error + // [] + let query = json!({ + "facetFilters": [] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + // [[]] + let query = json!({ + "facetFilters": [[]] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + // ["color:green", []] + let query = json!({ + "facetFilters": ["color:green", []] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + + // too much depth + // [[[]]] + let query = json!({ + "facetFilters": [[[]]] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + // [["color:green", ["color:blue"]]] + let query = json!({ + "facetFilters": [["color:green", ["color:blue"]]] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + // "color:green" + let query = json!({ + "facetFilters": "color:green" + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); +} + +#[actix_rt::test] +async fn placeholder_test_facet_count() { + let mut server = common::Server::test_server().await; + + // test without facet distribution + let query = json!({}); + test_post_get_search!(server, query, |response, _status_code| { + assert!(response.get("exhaustiveFacetsCount").is_none()); + assert!(response.get("facetsDistribution").is_none()); + }); + + // test no facets set, search on color + let query = json!({ + "facetsDistribution": ["color"] + }); + test_post_get_search!(server, query.clone(), |_response, status_code| { + assert_eq!(status_code, 400); + }); + + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + server.update_all_settings(body).await; + // same as before, but now facets are set: + test_post_get_search!(server, query, |response, _status_code| { + println!("{}", response); + assert!(response.get("exhaustiveFacetsCount").is_some()); + assert_eq!( + response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 1 + ); + }); + // searching on color and tags + let query = json!({ + "facetsDistribution": ["color", "tags"] + }); + test_post_get_search!(server, query, |response, _status_code| { + let facets = response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap(); + assert_eq!(facets.values().count(), 2); + assert_ne!( + !facets + .get("color") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 0 + ); + assert_ne!( + !facets + .get("tags") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 0 + ); + }); + // wildcard + let query = json!({ + "facetsDistribution": ["*"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_eq!( + response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 2 + ); + }); + // wildcard with other attributes: + let query = json!({ + "facetsDistribution": ["color", "*"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_eq!( + response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 2 + ); + }); + + // empty facet list + let query = json!({ + "facetsDistribution": [] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_eq!( + response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 0 + ); + }); + + // attr not set as facet passed: + let query = json!({ + "facetsDistribution": ["gender"] + }); + test_post_get_search!(server, query, |_response, status_code| { + assert_eq!(status_code, 400); + }); +} + +#[actix_rt::test] +#[should_panic] +async fn placeholder_test_bad_facet_distribution() { + let mut server = common::Server::test_server().await; + // string instead of array: + let query = json!({ + "facetsDistribution": "color" + }); + test_post_get_search!(server, query, |_response, _status_code| {}); + + // invalid value in array: + let query = json!({ + "facetsDistribution": ["color", true] + }); + test_post_get_search!(server, query, |_response, _status_code| {}); +} + +#[actix_rt::test] +async fn placeholder_test_sort() { + let mut server = common::Server::test_server().await; + + let body = json!({ + "rankingRules": ["asc(age)"], + "attributesForFaceting": ["color"] + }); + server.update_all_settings(body).await; + let query = json!({}); + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + hits.iter() + .map(|v| v["age"].as_u64().unwrap()) + .fold(0, |prev, cur| { + assert!(cur >= prev); + cur + }); + }); + + let query = json!({ + "facetFilters": ["color:green"] + }); + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + hits.iter() + .map(|v| v["age"].as_u64().unwrap()) + .fold(0, |prev, cur| { + assert!(cur >= prev); + cur + }); + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_empty_query() { + let mut server = common::Server::test_server().await; + + let query = json! ({ + "q": "", + "limit": 3 + }); + + test_post_get_search!(server, query, |response, status_code| { + eprintln!("{}", response); + assert_eq!(status_code, 200); + assert_eq!(response["hits"].as_array().unwrap().len(), 3); + }); +} + +#[actix_rt::test] +async fn test_filter_nb_hits_search_placeholder() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + let documents = json!([ + { + "id": 1, + "content": "a", + "color": "green", + "size": 1, + }, + { + "id": 2, + "content": "a", + "color": "green", + "size": 2, + }, + { + "id": 3, + "content": "a", + "color": "blue", + "size": 3, + }, + ]); + + server.add_or_update_multiple_documents(documents).await; + let (response, _) = server.search_post(json!({})).await; + assert_eq!(response["nbHits"], 3); + + server.update_distinct_attribute(json!("color")).await; + + let (response, _) = server.search_post(json!({})).await; + assert_eq!(response["nbHits"], 2); + + let (response, _) = server.search_post(json!({"filters": "size < 3"})).await; + println!("result: {}", response); + assert_eq!(response["nbHits"], 1); +} diff --git a/tests/search.rs b/tests/search.rs new file mode 100644 index 000000000..267c98265 --- /dev/null +++ b/tests/search.rs @@ -0,0 +1,1879 @@ +use std::convert::Into; + +use assert_json_diff::assert_json_eq; +use serde_json::json; +use serde_json::Value; + +#[macro_use] mod common; + +#[actix_rt::test] +async fn search() { + let mut server = common::Server::test_server().await; + + let query = json! ({ + "q": "exercitation" + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + }, + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 49, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ], + "isActive": true + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + let hits: Vec = hits.iter().cloned().take(3).collect(); + assert_json_eq!(expected.clone(), serde_json::to_value(hits).unwrap(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_no_params() { + let mut server = common::Server::test_server().await; + + let query = json! ({}); + + // an empty search should return the 20 first indexed document + let dataset: Vec = serde_json::from_slice(include_bytes!("assets/test_set.json")).unwrap(); + let expected: Vec = dataset.into_iter().take(20).collect(); + let expected: Value = serde_json::to_value(expected).unwrap(); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_in_unexisting_index() { + let mut server = common::Server::with_uid("test"); + + let query = json! ({ + "q": "exercitation" + }); + + let expected = json! ({ + "message": "Index test not found", + "errorCode": "index_not_found", + "errorType": "invalid_request_error", + "errorLink": "https://docs.meilisearch.com/errors#index_not_found" + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(404, status_code); + assert_json_eq!(expected.clone(), response.clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_unexpected_params() { + + let query = json! ({"lol": "unexpected"}); + + let expected = "unknown field `lol`, expected one of `q`, `offset`, `limit`, `attributesToRetrieve`, `attributesToCrop`, `cropLength`, `attributesToHighlight`, `filters`, `matches`, `facetFilters`, `facetsDistribution` at line 1 column 6"; + + let post_query = serde_json::from_str::(&query.to_string()); + assert!(post_query.is_err()); + assert_eq!(expected, post_query.err().unwrap().to_string()); + + let get_query: Result = serde_json::from_str(&query.to_string()); + assert!(get_query.is_err()); + assert_eq!(expected, get_query.err().unwrap().to_string()); +} + +#[actix_rt::test] +async fn search_with_limit() { + let mut server = common::Server::test_server().await; + + let query = json! ({ + "q": "exercitation", + "limit": 3 + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + }, + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 49, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ], + "isActive": true + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_offset() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 3, + "offset": 1 + }); + + let expected = json!([ + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 49, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ], + "isActive": true + }, + { + "id": 0, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ], + "isActive": false + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attribute_to_highlight_wildcard() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToHighlight": ["*"] + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attribute_to_highlight_1() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToHighlight": ["name"] + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_matches() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "matches": true + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_matchesInfo": { + "name": [ + { + "start": 0, + "length": 6 + } + ], + "email": [ + { + "start": 0, + "length": 6 + } + ] + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_crop() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 1, + "attributesToCrop": ["about"], + "cropLength": 20 + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attributes_to_retrieve() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","color","gender"], + }); + + let expected = json!([ + { + "name": "Cherry Orr", + "age": 27, + "color": "Green", + "gender": "female" + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attributes_to_retrieve_wildcard() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["*"], + }); + + let expected = json!([ + { + "id": 1, + "isActive": true, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ] + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_filter() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 3, + "filters": "gender='male'" + }); + + let expected = json!([ + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 0, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ], + "isActive": false + }, + { + "id": 66, + "balance": "$1,061.49", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "brown", + "name": "Higgins Aguilar", + "gender": "male", + "email": "higginsaguilar@chorizon.com", + "phone": "+1 (911) 540-3791", + "address": "132 Sackman Street, Layhill, Guam, 8729", + "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", + "registered": "2015-04-05T02:10:07 -02:00", + "latitude": 74.702813, + "longitude": 151.314972, + "tags": [ + "bug" + ], + "isActive": true + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 0, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ], + "isActive": false + } + ]); + + let query = json!({ + "q": "exercitation", + "limit": 3, + "filters": "name='Lucas Hess'" + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 2, + "balance": "$2,467.47", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", + "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", + "registered": "2014-10-28T12:59:30 -01:00", + "latitude": -64.008555, + "longitude": 11.867098, + "tags": [ + "good first issue" + ], + "isActive": true + }, + { + "id": 75, + "balance": "$1,913.42", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Emma Jacobs", + "gender": "female", + "email": "emmajacobs@chorizon.com", + "phone": "+1 (899) 554-3847", + "address": "173 Tapscott Street, Esmont, Maine, 7450", + "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", + "registered": "2019-03-29T06:24:13 -01:00", + "latitude": -35.53722, + "longitude": 155.703874, + "tags": [], + "isActive": false + } + ]); + let query = json!({ + "q": "exercitation", + "limit": 3, + "filters": "gender='female' AND (name='Patricia Goff' OR name='Emma Jacobs')" + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 30, + "balance": "$2,021.11", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Stacy Espinoza", + "gender": "female", + "email": "stacyespinoza@chorizon.com", + "phone": "+1 (999) 487-3253", + "address": "931 Alabama Avenue, Bangor, Alaska, 8215", + "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", + "registered": "2014-07-16T06:15:53 -02:00", + "latitude": 41.560197, + "longitude": 177.697, + "tags": [ + "new issue", + "new issue", + "bug" + ], + "isActive": true + }, + { + "id": 31, + "balance": "$3,609.82", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Vilma Garza", + "gender": "female", + "email": "vilmagarza@chorizon.com", + "phone": "+1 (944) 585-2021", + "address": "565 Tech Place, Sedley, Puerto Rico, 858", + "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", + "registered": "2017-06-30T07:43:52 -02:00", + "latitude": -12.574889, + "longitude": -54.771186, + "tags": [ + "new issue", + "wontfix", + "wontfix" + ], + "isActive": false + }, + { + "id": 2, + "balance": "$2,467.47", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", + "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", + "registered": "2014-10-28T12:59:30 -01:00", + "latitude": -64.008555, + "longitude": 11.867098, + "tags": [ + "good first issue" + ], + "isActive": true + } + ]); + let query = json!({ + "q": "exerciatation", + "limit": 3, + "filters": "gender='female' AND (name='Patricia Goff' OR age > 30)" + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 0, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ], + "isActive": false + }, + { + "id": 66, + "balance": "$1,061.49", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "brown", + "name": "Higgins Aguilar", + "gender": "male", + "email": "higginsaguilar@chorizon.com", + "phone": "+1 (911) 540-3791", + "address": "132 Sackman Street, Layhill, Guam, 8729", + "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", + "registered": "2015-04-05T02:10:07 -02:00", + "latitude": 74.702813, + "longitude": 151.314972, + "tags": [ + "bug" + ], + "isActive": true + } + ]); + let query = json!({ + "q": "exerciatation", + "limit": 3, + "filters": "NOT gender = 'female' AND age > 30" + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 11, + "balance": "$1,351.43", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Evans Wagner", + "gender": "male", + "email": "evanswagner@chorizon.com", + "phone": "+1 (889) 496-2332", + "address": "118 Monaco Place, Lutsen, Delaware, 6209", + "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", + "registered": "2016-10-27T01:26:31 -02:00", + "latitude": -77.673222, + "longitude": -142.657214, + "tags": [ + "good first issue", + "good first issue" + ], + "isActive": true + } + ]); + let query = json!({ + "q": "exerciatation", + "filters": "NOT gender = 'female' AND name='Evans Wagner'" + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attributes_to_highlight_and_matches() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToHighlight": ["name","email"], + "matches": true, + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + }, + "_matchesInfo": { + "email": [ + { + "start": 0, + "length": 6 + } + ], + "name": [ + { + "start": 0, + "length": 6 + } + ] + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attributes_to_highlight_and_matches_and_crop() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exerciatation", + "limit": 1, + "attributesToCrop": ["about"], + "cropLength": 20, + "attributesToHighlight": ["about"], + "matches": true, + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + }, + "_matchesInfo": { + "about": [ + { + "start": 0, + "length": 12 + } + ] + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","gender","email"], + "attributesToHighlight": ["name"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "name": "Cherry Orr" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_2() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 1, + "attributesToRetrieve": ["name","age","gender"], + "attributesToCrop": ["about"], + "cropLength": 20, + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "_formatted": { + "about": "Exercitation officia" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_3() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 1, + "attributesToRetrieve": ["name","age","gender"], + "attributesToCrop": ["about:20"], + }); + + let expected = json!( [ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "_formatted": { + "about": "Exercitation officia" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_4() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","email","gender"], + "attributesToCrop": ["name:0","email:6"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "name": "Cherry", + "email": "cherryorr" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_5() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","email","gender"], + "attributesToCrop": ["*","email:6"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "name": "Cherry Orr", + "email": "cherryorr", + "age": 27, + "gender": "female" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_6() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","email","gender"], + "attributesToCrop": ["*","email:10"], + "attributesToHighlight": ["name"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_7() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","gender","email"], + "attributesToCrop": ["*","email:6"], + "attributesToHighlight": ["*"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_8() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","email","gender","address"], + "attributesToCrop": ["*","email:6"], + "attributesToHighlight": ["*","address"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "_formatted": { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr", + "address": "442 Beverly Road, Ventress, New Mexico, 3361" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn test_faceted_search_valid() { + // set facetting attributes before adding documents + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test" })).await; + + let body = json!({ + "attributesForFaceting": ["color"] + }); + server.update_all_settings(body).await; + + let dataset = include_bytes!("assets/test_set.json"); + let body: Value = serde_json::from_slice(dataset).unwrap(); + server.add_or_update_multiple_documents(body).await; + + // simple tests on attributes with string value + + let query = json!({ + "q": "a", + "facetFilters": ["color:green"] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "Green")); + }); + + let query = json!({ + "q": "a", + "facetFilters": [["color:blue"]] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue")); + }); + + let query = json!({ + "q": "a", + "facetFilters": ["color:Blue"] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue")); + }); + + // test on arrays: ["tags:bug"] + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + + server.update_all_settings(body).await; + + let query = json!({ + "q": "a", + "facetFilters": ["tags:bug"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); + }); + + // test and: ["color:blue", "tags:bug"] + let query = json!({ + "q": "a", + "facetFilters": ["color:blue", "tags:bug"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value + .get("color") + .unwrap() == "blue" + && value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); + }); + + // test or: [["color:blue", "color:green"]] + let query = json!({ + "q": "a", + "facetFilters": [["color:blue", "color:green"]] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| + value + .get("color") + .unwrap() == "blue" + || value + .get("color") + .unwrap() == "Green")); + }); + // test and-or: ["tags:bug", ["color:blue", "color:green"]] + let query = json!({ + "q": "a", + "facetFilters": ["tags:bug", ["color:blue", "color:green"]] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| + value + .get("tags") + .unwrap() + .as_array() + .unwrap() + .contains(&Value::String("bug".to_owned())) + && (value + .get("color") + .unwrap() == "blue" + || value + .get("color") + .unwrap() == "Green"))); + + }); +} + +#[actix_rt::test] +async fn test_faceted_search_invalid() { + let mut server = common::Server::test_server().await; + + //no faceted attributes set + let query = json!({ + "q": "a", + "facetFilters": ["color:blue"] + }); + + test_post_get_search!(server, query, |response, status_code| { + + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + server.update_all_settings(body).await; + // empty arrays are error + // [] + let query = json!({ + "q": "a", + "facetFilters": [] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + // [[]] + let query = json!({ + "q": "a", + "facetFilters": [[]] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + // ["color:green", []] + let query = json!({ + "q": "a", + "facetFilters": ["color:green", []] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + // too much depth + // [[[]]] + let query = json!({ + "q": "a", + "facetFilters": [[[]]] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + // [["color:green", ["color:blue"]]] + let query = json!({ + "q": "a", + "facetFilters": [["color:green", ["color:blue"]]] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + // "color:green" + let query = json!({ + "q": "a", + "facetFilters": "color:green" + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); +} + +#[actix_rt::test] +async fn test_facet_count() { + let mut server = common::Server::test_server().await; + + // test without facet distribution + let query = json!({ + "q": "a", + }); + test_post_get_search!(server, query, |response, _status_code|{ + assert!(response.get("exhaustiveFacetsCount").is_none()); + assert!(response.get("facetsDistribution").is_none()); + }); + + // test no facets set, search on color + let query = json!({ + "q": "a", + "facetsDistribution": ["color"] + }); + test_post_get_search!(server, query.clone(), |_response, status_code|{ + assert_eq!(status_code, 400); + }); + + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + server.update_all_settings(body).await; + // same as before, but now facets are set: + test_post_get_search!(server, query, |response, _status_code|{ + println!("{}", response); + assert!(response.get("exhaustiveFacetsCount").is_some()); + assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 1); + // assert that case is preserved + assert!(response["facetsDistribution"] + .as_object() + .unwrap()["color"] + .as_object() + .unwrap() + .get("Green") + .is_some()); + }); + // searching on color and tags + let query = json!({ + "q": "a", + "facetsDistribution": ["color", "tags"] + }); + test_post_get_search!(server, query, |response, _status_code|{ + let facets = response.get("facetsDistribution").unwrap().as_object().unwrap(); + assert_eq!(facets.values().count(), 2); + assert_ne!(!facets.get("color").unwrap().as_object().unwrap().values().count(), 0); + assert_ne!(!facets.get("tags").unwrap().as_object().unwrap().values().count(), 0); + }); + // wildcard + let query = json!({ + "q": "a", + "facetsDistribution": ["*"] + }); + test_post_get_search!(server, query, |response, _status_code|{ + assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); + }); + // wildcard with other attributes: + let query = json!({ + "q": "a", + "facetsDistribution": ["color", "*"] + }); + test_post_get_search!(server, query, |response, _status_code|{ + assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); + }); + + // empty facet list + let query = json!({ + "q": "a", + "facetsDistribution": [] + }); + test_post_get_search!(server, query, |response, _status_code|{ + assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 0); + }); + + // attr not set as facet passed: + let query = json!({ + "q": "a", + "facetsDistribution": ["gender"] + }); + test_post_get_search!(server, query, |_response, status_code|{ + assert_eq!(status_code, 400); + }); + +} + +#[actix_rt::test] +#[should_panic] +async fn test_bad_facet_distribution() { + let mut server = common::Server::test_server().await; + // string instead of array: + let query = json!({ + "q": "a", + "facetsDistribution": "color" + }); + test_post_get_search!(server, query, |_response, _status_code| {}); + + // invalid value in array: + let query = json!({ + "q": "a", + "facetsDistribution": ["color", true] + }); + test_post_get_search!(server, query, |_response, _status_code| {}); +} + +#[actix_rt::test] +async fn highlight_cropped_text() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + let doc = json!([ + { + "id": 1, + "body": r##"well, it may not work like that, try the following: +1. insert your trip +2. google your `searchQuery` +3. find a solution +> say hello"## + } + ]); + server.add_or_replace_multiple_documents(doc).await; + + // tests from #680 + //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=30"; + let query = json!({ + "q": "insert", + "attributesToHighlight": ["*"], + "attributesToCrop": ["body"], + "cropLength": 30, + }); + let expected_response = "that, try the following: \n1. insert your trip\n2. google your"; + test_post_get_search!(server, query, |response, _status_code|{ + assert_eq!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .get(0) + .unwrap() + .as_object() + .unwrap() + .get("_formatted") + .unwrap() + .as_object() + .unwrap() + .get("body") + .unwrap() + , &Value::String(expected_response.to_owned())); + }); + + //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=80"; + let query = json!({ + "q": "insert", + "attributesToHighlight": ["*"], + "attributesToCrop": ["body"], + "cropLength": 80, + }); + let expected_response = "well, it may not work like that, try the following: \n1. insert your trip\n2. google your `searchQuery`\n3. find a solution \n> say hello"; + test_post_get_search!(server, query, |response, _status_code| { + assert_eq!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .get(0) + .unwrap() + .as_object() + .unwrap() + .get("_formatted") + .unwrap() + .as_object() + .unwrap() + .get("body") + .unwrap() + , &Value::String(expected_response.to_owned())); + }); +} + +#[actix_rt::test] +async fn well_formated_error_with_bad_request_params() { + let mut server = common::Server::with_uid("test"); + let query = "foo=bar"; + let (response, _status_code) = server.search_get(query).await; + assert!(response.get("message").is_some()); + assert!(response.get("errorCode").is_some()); + assert!(response.get("errorType").is_some()); + assert!(response.get("errorLink").is_some()); +} + + +#[actix_rt::test] +async fn update_documents_with_facet_distribution() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + let settings = json!({ + "attributesForFaceting": ["genre"], + "displayedAttributes": ["genre"], + "searchableAttributes": ["genre"] + }); + server.update_all_settings(settings).await; + let update1 = json!([ + { + "id": "1", + "type": "album", + "title": "Nevermind", + "genre": ["grunge", "alternative"] + }, + { + "id": "2", + "type": "album", + "title": "Mellon Collie and the Infinite Sadness", + "genre": ["alternative", "rock"] + }, + { + "id": "3", + "type": "album", + "title": "The Queen Is Dead", + "genre": ["indie", "rock"] + } + ]); + server.add_or_update_multiple_documents(update1).await; + let search = json!({ + "q": "album", + "facetsDistribution": ["genre"] + }); + let (response1, _) = server.search_post(search.clone()).await; + let expected_facet_distribution = json!({ + "genre": { + "grunge": 1, + "alternative": 2, + "rock": 2, + "indie": 1 + } + }); + assert_json_eq!(expected_facet_distribution.clone(), response1["facetsDistribution"].clone()); + + let update2 = json!([ + { + "id": "3", + "title": "The Queen Is Very Dead" + } + ]); + server.add_or_update_multiple_documents(update2).await; + let (response2, _) = server.search_post(search).await; + assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone()); +} + +#[actix_rt::test] +async fn test_filter_nb_hits_search_normal() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + let documents = json!([ + { + "id": 1, + "content": "a", + "color": "green", + "size": 1, + }, + { + "id": 2, + "content": "a", + "color": "green", + "size": 2, + }, + { + "id": 3, + "content": "a", + "color": "blue", + "size": 3, + }, + ]); + + server.add_or_update_multiple_documents(documents).await; + let (response, _) = server.search_post(json!({"q": "a"})).await; + assert_eq!(response["nbHits"], 3); + + let (response, _) = server.search_post(json!({"q": "a", "filters": "size = 1"})).await; + assert_eq!(response["nbHits"], 1); + + server.update_distinct_attribute(json!("color")).await; + + let (response, _) = server.search_post(json!({"q": "a"})).await; + assert_eq!(response["nbHits"], 2); + + let (response, _) = server.search_post(json!({"q": "a", "filters": "size < 3"})).await; + println!("result: {}", response); + assert_eq!(response["nbHits"], 1); +} diff --git a/tests/search_settings.rs b/tests/search_settings.rs new file mode 100644 index 000000000..46417498d --- /dev/null +++ b/tests/search_settings.rs @@ -0,0 +1,538 @@ +use assert_json_diff::assert_json_eq; +use serde_json::json; +use std::convert::Into; + +mod common; + +#[actix_rt::test] +async fn search_with_settings_basic() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "age", + "color", + "gender", + "email", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": null, + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=ea%20exercitation&limit=3"; + + let expect = json!([ + { + "balance": "$2,467.47", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700" + }, + { + "balance": "$3,344.40", + "age": 35, + "color": "blue", + "name": "Adeline Flynn", + "gender": "female", + "email": "adelineflynn@chorizon.com", + "phone": "+1 (994) 600-2840", + "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948" + }, + { + "balance": "$3,394.96", + "age": 25, + "color": "blue", + "name": "Aida Kirby", + "gender": "female", + "email": "aidakirby@chorizon.com", + "phone": "+1 (942) 532-2325", + "address": "797 Engert Avenue, Wilsonia, Idaho, 6532" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_stop_words() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "age", + "color", + "gender", + "email", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": ["ea"], + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=ea%20exercitation&limit=3"; + let expect = json!([ + { + "balance": "$1,921.58", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219" + }, + { + "balance": "$1,706.13", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361" + }, + { + "balance": "$1,476.39", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_synonyms() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "age", + "color", + "gender", + "email", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": null, + "synonyms": { + "application": [ + "exercitation" + ] + }, + }); + + server.update_all_settings(config).await; + + let query = "q=application&limit=3"; + let expect = json!([ + { + "balance": "$1,921.58", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219" + }, + { + "balance": "$1,706.13", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361" + }, + { + "balance": "$1,476.39", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_ranking_rules() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "age", + "color", + "gender", + "email", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": null, + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=exarcitation&limit=3"; + let expect = json!([ + { + "balance": "$1,921.58", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219" + }, + { + "balance": "$1,706.13", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361" + }, + { + "balance": "$1,476.39", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + println!("{}", response["hits"].clone()); + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_searchable_attributes() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "age", + "color", + "gender", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": null, + "synonyms": { + "exarcitation": [ + "exercitation" + ] + }, + }); + + server.update_all_settings(config).await; + + let query = "q=Carol&limit=3"; + let expect = json!([ + { + "balance": "$1,440.09", + "age": 40, + "color": "blue", + "name": "Levy Whitley", + "gender": "male", + "email": "levywhitley@chorizon.com", + "phone": "+1 (911) 458-2411", + "address": "187 Thomas Street, Hachita, North Carolina, 2989" + }, + { + "balance": "$1,977.66", + "age": 36, + "color": "brown", + "name": "Combs Stanley", + "gender": "male", + "email": "combsstanley@chorizon.com", + "phone": "+1 (827) 419-2053", + "address": "153 Beverley Road, Siglerville, South Carolina, 3666" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_displayed_attributes() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "age", + "color", + "gender", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone" + ], + "stopWords": null, + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=exercitation&limit=3"; + let expect = json!([ + { + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243" + }, + { + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174" + }, + { + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_searchable_attributes_2() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "age", + "color", + "gender", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender" + ], + "stopWords": null, + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=exercitation&limit=3"; + let expect = json!([ + { + "age": 31, + "name": "Harper Carson", + "gender": "male" + }, + { + "age": 27, + "name": "Cherry Orr", + "gender": "female" + }, + { + "age": 28, + "name": "Maureen Dale", + "gender": "female" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +// issue #798 +#[actix_rt::test] +async fn distinct_attributes_returns_name_not_id() { + let mut server = common::Server::test_server().await; + let settings = json!({ + "distinctAttribute": "color", + }); + server.update_all_settings(settings).await; + let (response, _) = server.get_all_settings().await; + assert_eq!(response["distinctAttribute"], "color"); + let (response, _) = server.get_distinct_attribute().await; + assert_eq!(response, "color"); +} diff --git a/tests/settings.rs b/tests/settings.rs new file mode 100644 index 000000000..6b125c13a --- /dev/null +++ b/tests/settings.rs @@ -0,0 +1,523 @@ +use assert_json_diff::assert_json_eq; +use serde_json::json; +use std::convert::Into; +mod common; + +#[actix_rt::test] +async fn write_all_and_delete() { + let mut server = common::Server::test_server().await; + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["name"], + }); + + server.update_all_settings(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: false); + + // 4 - Delete all settings + + server.delete_all_settings().await; + + // 5 - Get all settings and check if they are set to default values + + let (response, _status_code) = server.get_all_settings().await; + + let expect = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": ["*"], + "displayedAttributes": ["*"], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [], + }); + + assert_json_eq!(expect, response, ordered: false); +} + +#[actix_rt::test] +async fn write_all_and_update() { + let mut server = common::Server::test_server().await; + + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["name"], + }); + + server.update_all_settings(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: false); + + // 4 - Update all settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(age)", + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "color", + "age", + ], + "displayedAttributes": [ + "name", + "color", + "age", + "registered", + "picture", + ], + "stopWords": [], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["title"], + }); + + server.update_all_settings(body).await; + + // 5 - Get all settings and check if the content is the same of (4) + + let (response, _status_code) = server.get_all_settings().await; + + let expected = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(age)", + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "color", + "age", + ], + "displayedAttributes": [ + "name", + "color", + "age", + "registered", + "picture", + ], + "stopWords": [], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["title"], + }); + + assert_json_eq!(expected, response, ordered: false); +} + +#[actix_rt::test] +async fn test_default_settings() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + }); + server.create_index(body).await; + + // 1 - Get all settings and compare to the previous one + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": ["*"], + "displayedAttributes": ["*"], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [], + }); + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: false); +} + +#[actix_rt::test] +async fn test_default_settings_2() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + // 1 - Get all settings and compare to the previous one + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": ["*"], + "displayedAttributes": ["*"], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [], + }); + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: false); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/516 +#[actix_rt::test] +async fn write_setting_and_update_partial() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + }); + server.create_index(body).await; + + // 2 - Send the settings + + let body = json!({ + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ] + }); + + server.update_all_settings(body.clone()).await; + + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(age)", + "desc(registered)", + ], + "distinctAttribute": "id", + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + }); + + server.update_all_settings(body.clone()).await; + + // 2 - Send the settings + + let expected = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(age)", + "desc(registered)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": [], + }); + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(expected, response, ordered: false); +} + +#[actix_rt::test] +async fn attributes_for_faceting_settings() { + let mut server = common::Server::test_server().await; + // initial attributes array should be empty + let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; + assert_eq!(response, json!([])); + // add an attribute and test for its presence + let (_response, _status_code) = server.post_request_async( + "/indexes/test/settings/attributes-for-faceting", + json!(["foobar"])).await; + let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; + assert_eq!(response, json!(["foobar"])); + // remove all attributes and test for emptiness + let (_response, _status_code) = server.delete_request_async( + "/indexes/test/settings/attributes-for-faceting").await; + let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; + assert_eq!(response, json!([])); +} + +#[actix_rt::test] +async fn setting_ranking_rules_dont_mess_with_other_settings() { + let mut server = common::Server::test_server().await; + let body = json!({ + "rankingRules": ["asc(foobar)"] + }); + server.update_all_settings(body).await; + let (response, _) = server.get_all_settings().await; + assert_eq!(response["rankingRules"].as_array().unwrap().len(), 1); + assert_eq!(response["rankingRules"].as_array().unwrap().first().unwrap().as_str().unwrap(), "asc(foobar)"); + assert!(!response["searchableAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); + assert!(!response["displayedAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); +} + +#[actix_rt::test] +async fn displayed_and_searchable_attributes_reset_to_wildcard() { + let mut server = common::Server::test_server().await; + server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; + let (response, _) = server.get_all_settings().await; + + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); + + server.delete_searchable_attributes().await; + server.delete_displayed_attributes().await; + + let (response, _) = server.get_all_settings().await; + + assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); + + let mut server = common::Server::test_server().await; + server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; + let (response, _) = server.get_all_settings().await; + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); + + server.update_all_settings(json!({ "searchableAttributes": [], "displayedAttributes": [] })).await; + + let (response, _) = server.get_all_settings().await; + + assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); +} + +#[actix_rt::test] +async fn settings_that_contains_wildcard_is_wildcard() { + let mut server = common::Server::test_server().await; + server.update_all_settings(json!({ "searchableAttributes": ["color", "*"], "displayedAttributes": ["color", "*"] })).await; + + let (response, _) = server.get_all_settings().await; + + assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); +} + +#[actix_rt::test] +async fn test_displayed_attributes_field() { + let mut server = common::Server::test_server().await; + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "age", + "email", + "gender", + "name", + "registered", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["avenue", "street"], + "street": ["avenue"], + }, + "attributesForFaceting": ["name"], + }); + + server.update_all_settings(body.clone()).await; + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: true); +} \ No newline at end of file diff --git a/tests/settings_ranking_rules.rs b/tests/settings_ranking_rules.rs new file mode 100644 index 000000000..ac9a1e00c --- /dev/null +++ b/tests/settings_ranking_rules.rs @@ -0,0 +1,182 @@ +use assert_json_diff::assert_json_eq; +use serde_json::json; + +mod common; + +#[actix_rt::test] +async fn write_all_and_delete() { + let mut server = common::Server::test_server().await; + + // 2 - Send the settings + + let body = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ]); + + server.update_ranking_rules(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (response, _status_code) = server.get_ranking_rules().await; + + assert_json_eq!(body, response, ordered: false); + + // 4 - Delete all settings + + server.delete_ranking_rules().await; + + // 5 - Get all settings and check if they are empty + + let (response, _status_code) = server.get_ranking_rules().await; + + let expected = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ]); + + assert_json_eq!(expected, response, ordered: false); +} + +#[actix_rt::test] +async fn write_all_and_update() { + let mut server = common::Server::test_server().await; + + // 2 - Send the settings + + let body = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ]); + + server.update_ranking_rules(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (response, _status_code) = server.get_ranking_rules().await; + + assert_json_eq!(body, response, ordered: false); + + // 4 - Update all settings + + let body = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + ]); + + server.update_ranking_rules(body).await; + + // 5 - Get all settings and check if the content is the same of (4) + + let (response, _status_code) = server.get_ranking_rules().await; + + let expected = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + ]); + + assert_json_eq!(expected, response, ordered: false); +} + +#[actix_rt::test] +async fn send_undefined_rule() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + let body = json!(["typos",]); + + let (_response, status_code) = server.update_ranking_rules_sync(body).await; + assert_eq!(status_code, 400); +} + +#[actix_rt::test] +async fn send_malformed_custom_rule() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + let body = json!(["dsc(truc)",]); + + let (_response, status_code) = server.update_ranking_rules_sync(body).await; + assert_eq!(status_code, 400); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/521 +#[actix_rt::test] +async fn write_custom_ranking_and_index_documents() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + // 1 - Add ranking rules with one custom ranking on a string + + let body = json!(["asc(name)", "typo"]); + + server.update_ranking_rules(body).await; + + // 2 - Add documents + + let body = json!([ + { + "id": 1, + "name": "Cherry Orr", + "color": "green" + }, + { + "id": 2, + "name": "Lucas Hess", + "color": "yellow" + } + ]); + + server.add_or_replace_multiple_documents(body).await; + + // 3 - Get the first document and compare + + let expected = json!({ + "id": 1, + "name": "Cherry Orr", + "color": "green" + }); + + let (response, status_code) = server.get_document(1).await; + assert_eq!(status_code, 200); + + assert_json_eq!(response, expected, ordered: false); +} diff --git a/tests/settings_stop_words.rs b/tests/settings_stop_words.rs new file mode 100644 index 000000000..3ff2e8bb7 --- /dev/null +++ b/tests/settings_stop_words.rs @@ -0,0 +1,61 @@ +use assert_json_diff::assert_json_eq; +use serde_json::json; + +mod common; + +#[actix_rt::test] +async fn update_stop_words() { + let mut server = common::Server::test_server().await; + + // 1 - Get stop words + + let (response, _status_code) = server.get_stop_words().await; + assert_eq!(response.as_array().unwrap().is_empty(), true); + + // 2 - Update stop words + + let body = json!(["ut", "ea"]); + server.update_stop_words(body.clone()).await; + + // 3 - Get all stop words and compare to the previous one + + let (response, _status_code) = server.get_stop_words().await; + assert_json_eq!(body, response, ordered: false); + + // 4 - Delete all stop words + + server.delete_stop_words().await; + + // 5 - Get all stop words and check if they are empty + + let (response, _status_code) = server.get_stop_words().await; + assert_eq!(response.as_array().unwrap().is_empty(), true); +} + +#[actix_rt::test] +async fn add_documents_and_stop_words() { + let mut server = common::Server::test_server().await; + + // 2 - Update stop words + + let body = json!(["ad", "in"]); + server.update_stop_words(body.clone()).await; + + // 3 - Search for a document with stop words + + let (response, _status_code) = server.search_get("q=in%20exercitation").await; + assert!(!response["hits"].as_array().unwrap().is_empty()); + + // 4 - Search for documents with *only* stop words + + let (response, _status_code) = server.search_get("q=ad%20in").await; + assert!(response["hits"].as_array().unwrap().is_empty()); + + // 5 - Delete all stop words + + // server.delete_stop_words(); + + // // 6 - Search for a document with one stop word + + // assert!(!response["hits"].as_array().unwrap().is_empty()); +} diff --git a/tests/url_normalizer.rs b/tests/url_normalizer.rs new file mode 100644 index 000000000..c2c9187ee --- /dev/null +++ b/tests/url_normalizer.rs @@ -0,0 +1,18 @@ +mod common; + +#[actix_rt::test] +async fn url_normalizer() { + let mut server = common::Server::with_uid("movies"); + + let (_response, status_code) = server.get_request("/version").await; + assert_eq!(status_code, 200); + + let (_response, status_code) = server.get_request("//version").await; + assert_eq!(status_code, 200); + + let (_response, status_code) = server.get_request("/version/").await; + assert_eq!(status_code, 200); + + let (_response, status_code) = server.get_request("//version/").await; + assert_eq!(status_code, 200); +} From 29b1f55bb04a902861bca9111ce18233bd6b185a Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 12 Dec 2020 16:04:37 +0100 Subject: [PATCH 002/522] prepare boilerplate code for new api --- src/helpers/meilisearch.rs | 652 --------------------------------- src/helpers/mod.rs | 1 - src/models/mod.rs | 1 - src/models/update_operation.rs | 33 -- src/routes/document.rs | 136 +------ src/routes/dump.rs | 26 +- src/routes/index.rs | 259 +------------ src/routes/key.rs | 6 +- src/routes/search.rs | 175 +-------- src/routes/setting.rs | 355 +----------------- src/routes/stats.rs | 78 +--- src/routes/stop_words.rs | 38 +- src/routes/synonym.rs | 47 +-- 13 files changed, 50 insertions(+), 1757 deletions(-) delete mode 100644 src/helpers/meilisearch.rs delete mode 100644 src/models/mod.rs delete mode 100644 src/models/update_operation.rs diff --git a/src/helpers/meilisearch.rs b/src/helpers/meilisearch.rs deleted file mode 100644 index 749fe410e..000000000 --- a/src/helpers/meilisearch.rs +++ /dev/null @@ -1,652 +0,0 @@ -use std::cmp::Ordering; -use std::collections::{HashMap, HashSet}; -use std::hash::{Hash, Hasher}; -use std::time::Instant; - -use indexmap::IndexMap; -use log::error; -use meilisearch_core::{Filter, MainReader}; -use meilisearch_core::facets::FacetFilter; -use meilisearch_core::criterion::*; -use meilisearch_core::settings::RankingRule; -use meilisearch_core::{Highlight, Index, RankedMap}; -use meilisearch_schema::{FieldId, Schema}; -use meilisearch_tokenizer::is_cjk; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use siphasher::sip::SipHasher; -use slice_group_by::GroupBy; - -use crate::error::{Error, ResponseError}; - -pub trait IndexSearchExt { - fn new_search(&self, query: Option) -> SearchBuilder; -} - -impl IndexSearchExt for Index { - fn new_search(&self, query: Option) -> SearchBuilder { - SearchBuilder { - index: self, - query, - offset: 0, - limit: 20, - attributes_to_crop: None, - attributes_to_retrieve: None, - attributes_to_highlight: None, - filters: None, - matches: false, - facet_filters: None, - facets: None, - } - } -} - -pub struct SearchBuilder<'a> { - index: &'a Index, - query: Option, - offset: usize, - limit: usize, - attributes_to_crop: Option>, - attributes_to_retrieve: Option>, - attributes_to_highlight: Option>, - filters: Option, - matches: bool, - facet_filters: Option, - facets: Option> -} - -impl<'a> SearchBuilder<'a> { - pub fn offset(&mut self, value: usize) -> &SearchBuilder { - self.offset = value; - self - } - - pub fn limit(&mut self, value: usize) -> &SearchBuilder { - self.limit = value; - self - } - - pub fn attributes_to_crop(&mut self, value: HashMap) -> &SearchBuilder { - self.attributes_to_crop = Some(value); - self - } - - pub fn attributes_to_retrieve(&mut self, value: HashSet) -> &SearchBuilder { - self.attributes_to_retrieve = Some(value); - self - } - - pub fn add_retrievable_field(&mut self, value: String) -> &SearchBuilder { - let attributes_to_retrieve = self.attributes_to_retrieve.get_or_insert(HashSet::new()); - attributes_to_retrieve.insert(value); - self - } - - pub fn attributes_to_highlight(&mut self, value: HashSet) -> &SearchBuilder { - self.attributes_to_highlight = Some(value); - self - } - - pub fn add_facet_filters(&mut self, filters: FacetFilter) -> &SearchBuilder { - self.facet_filters = Some(filters); - self - } - - pub fn filters(&mut self, value: String) -> &SearchBuilder { - self.filters = Some(value); - self - } - - pub fn get_matches(&mut self) -> &SearchBuilder { - self.matches = true; - self - } - - pub fn add_facets(&mut self, facets: Vec<(FieldId, String)>) -> &SearchBuilder { - self.facets = Some(facets); - self - } - - pub fn search(self, reader: &MainReader) -> Result { - let schema = self - .index - .main - .schema(reader)? - .ok_or(Error::internal("missing schema"))?; - - let ranked_map = self.index.main.ranked_map(reader)?.unwrap_or_default(); - - // Change criteria - let mut query_builder = match self.get_criteria(reader, &ranked_map, &schema)? { - Some(criteria) => self.index.query_builder_with_criteria(criteria), - None => self.index.query_builder(), - }; - - if let Some(filter_expression) = &self.filters { - let filter = Filter::parse(filter_expression, &schema)?; - let index = &self.index; - query_builder.with_filter(move |id| { - let reader = &reader; - let filter = &filter; - match filter.test(reader, index, id) { - Ok(res) => res, - Err(e) => { - log::warn!("unexpected error during filtering: {}", e); - false - } - } - }); - } - - if let Some(field) = self.index.main.distinct_attribute(reader)? { - let index = &self.index; - query_builder.with_distinct(1, move |id| { - match index.document_attribute_bytes(reader, id, field) { - Ok(Some(bytes)) => { - let mut s = SipHasher::new(); - bytes.hash(&mut s); - Some(s.finish()) - } - _ => None, - } - }); - } - - query_builder.set_facet_filter(self.facet_filters); - query_builder.set_facets(self.facets); - - let start = Instant::now(); - let result = query_builder.query(reader, self.query.as_deref(), self.offset..(self.offset + self.limit)); - let search_result = result.map_err(Error::search_documents)?; - let time_ms = start.elapsed().as_millis() as usize; - - let mut all_attributes: HashSet<&str> = HashSet::new(); - let mut all_formatted: HashSet<&str> = HashSet::new(); - - match &self.attributes_to_retrieve { - Some(to_retrieve) => { - all_attributes.extend(to_retrieve.iter().map(String::as_str)); - - if let Some(to_highlight) = &self.attributes_to_highlight { - all_formatted.extend(to_highlight.iter().map(String::as_str)); - } - - if let Some(to_crop) = &self.attributes_to_crop { - all_formatted.extend(to_crop.keys().map(String::as_str)); - } - - all_attributes.extend(&all_formatted); - }, - None => { - all_attributes.extend(schema.displayed_name()); - // If we specified at least one attribute to highlight or crop then - // all available attributes will be returned in the _formatted field. - if self.attributes_to_highlight.is_some() || self.attributes_to_crop.is_some() { - all_formatted.extend(all_attributes.iter().cloned()); - } - }, - } - - let mut hits = Vec::with_capacity(self.limit); - for doc in search_result.documents { - let mut document: IndexMap = self - .index - .document(reader, Some(&all_attributes), doc.id) - .map_err(|e| Error::retrieve_document(doc.id.0, e))? - .ok_or(Error::internal( - "Impossible to retrieve the document; Corrupted data", - ))?; - - let mut formatted = document.iter() - .filter(|(key, _)| all_formatted.contains(key.as_str())) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut matches = doc.highlights.clone(); - - // Crops fields if needed - if let Some(fields) = &self.attributes_to_crop { - crop_document(&mut formatted, &mut matches, &schema, fields); - } - - // Transform to readable matches - if let Some(attributes_to_highlight) = &self.attributes_to_highlight { - let matches = calculate_matches( - &matches, - self.attributes_to_highlight.clone(), - &schema, - ); - formatted = calculate_highlights(&formatted, &matches, attributes_to_highlight); - } - - let matches_info = if self.matches { - Some(calculate_matches(&matches, self.attributes_to_retrieve.clone(), &schema)) - } else { - None - }; - - if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve { - document.retain(|key, _| attributes_to_retrieve.contains(&key.to_string())) - } - - let hit = SearchHit { - document, - formatted, - matches_info, - }; - - hits.push(hit); - } - - let results = SearchResult { - hits, - offset: self.offset, - limit: self.limit, - nb_hits: search_result.nb_hits, - exhaustive_nb_hits: search_result.exhaustive_nb_hit, - processing_time_ms: time_ms, - query: self.query.unwrap_or_default(), - facets_distribution: search_result.facets, - exhaustive_facets_count: search_result.exhaustive_facets_count, - }; - - Ok(results) - } - - pub fn get_criteria( - &self, - reader: &MainReader, - ranked_map: &'a RankedMap, - schema: &Schema, - ) -> Result>, ResponseError> { - let ranking_rules = self.index.main.ranking_rules(reader)?; - - if let Some(ranking_rules) = ranking_rules { - let mut builder = CriteriaBuilder::with_capacity(7 + ranking_rules.len()); - for rule in ranking_rules { - match rule { - RankingRule::Typo => builder.push(Typo), - RankingRule::Words => builder.push(Words), - RankingRule::Proximity => builder.push(Proximity), - RankingRule::Attribute => builder.push(Attribute), - RankingRule::WordsPosition => builder.push(WordsPosition), - RankingRule::Exactness => builder.push(Exactness), - RankingRule::Asc(field) => { - match SortByAttr::lower_is_better(&ranked_map, &schema, &field) { - Ok(rule) => builder.push(rule), - Err(err) => error!("Error during criteria builder; {:?}", err), - } - } - RankingRule::Desc(field) => { - match SortByAttr::higher_is_better(&ranked_map, &schema, &field) { - Ok(rule) => builder.push(rule), - Err(err) => error!("Error during criteria builder; {:?}", err), - } - } - } - } - builder.push(DocumentId); - return Ok(Some(builder.build())); - } - - Ok(None) - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct MatchPosition { - pub start: usize, - pub length: usize, -} - -impl PartialOrd for MatchPosition { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for MatchPosition { - fn cmp(&self, other: &Self) -> Ordering { - match self.start.cmp(&other.start) { - Ordering::Equal => self.length.cmp(&other.length), - _ => self.start.cmp(&other.start), - } - } -} - -pub type HighlightInfos = HashMap; -pub type MatchesInfos = HashMap>; -// pub type RankingInfos = HashMap; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SearchHit { - #[serde(flatten)] - pub document: IndexMap, - #[serde(rename = "_formatted", skip_serializing_if = "IndexMap::is_empty")] - pub formatted: IndexMap, - #[serde(rename = "_matchesInfo", skip_serializing_if = "Option::is_none")] - pub matches_info: Option, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SearchResult { - pub hits: Vec, - pub offset: usize, - pub limit: usize, - pub nb_hits: usize, - pub exhaustive_nb_hits: bool, - pub processing_time_ms: usize, - pub query: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub facets_distribution: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub exhaustive_facets_count: Option, -} - -/// returns the start index and the length on the crop. -fn aligned_crop(text: &str, match_index: usize, context: usize) -> (usize, usize) { - let is_word_component = |c: &char| c.is_alphanumeric() && !is_cjk(*c); - - let word_end_index = |mut index| { - if text.chars().nth(index - 1).map_or(false, |c| is_word_component(&c)) { - index += text.chars().skip(index).take_while(is_word_component).count(); - } - index - }; - - if context == 0 { - // count need to be at least 1 for cjk queries to return something - return (match_index, 1 + text.chars().skip(match_index).take_while(is_word_component).count()); - } - let start = match match_index.saturating_sub(context) { - 0 => 0, - n => { - let word_end_index = word_end_index(n); - // skip whitespaces if any - word_end_index + text.chars().skip(word_end_index).take_while(char::is_ascii_whitespace).count() - } - }; - let end = word_end_index(match_index + context); - - (start, end - start) -} - -fn crop_text( - text: &str, - matches: impl IntoIterator, - context: usize, -) -> (String, Vec) { - let mut matches = matches.into_iter().peekable(); - - let char_index = matches.peek().map(|m| m.char_index as usize).unwrap_or(0); - let (start, count) = aligned_crop(text, char_index, context); - - // TODO do something about double allocation - let text = text - .chars() - .skip(start) - .take(count) - .collect::() - .trim() - .to_string(); - - // update matches index to match the new cropped text - let matches = matches - .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + count) - .map(|m| Highlight { - char_index: m.char_index - start as u16, - ..m - }) - .collect(); - - (text, matches) -} - -fn crop_document( - document: &mut IndexMap, - matches: &mut Vec, - schema: &Schema, - fields: &HashMap, -) { - matches.sort_unstable_by_key(|m| (m.char_index, m.char_length)); - - for (field, length) in fields { - let attribute = match schema.id(field) { - Some(attribute) => attribute, - None => continue, - }; - - let selected_matches = matches - .iter() - .filter(|m| FieldId::new(m.attribute) == attribute) - .cloned(); - - if let Some(Value::String(ref mut original_text)) = document.get_mut(field) { - let (cropped_text, cropped_matches) = - crop_text(original_text, selected_matches, *length); - - *original_text = cropped_text; - - matches.retain(|m| FieldId::new(m.attribute) != attribute); - matches.extend_from_slice(&cropped_matches); - } - } -} - -fn calculate_matches( - matches: &[Highlight], - attributes_to_retrieve: Option>, - schema: &Schema, -) -> MatchesInfos { - let mut matches_result: HashMap> = HashMap::new(); - for m in matches.iter() { - if let Some(attribute) = schema.name(FieldId::new(m.attribute)) { - if let Some(ref attributes_to_retrieve) = attributes_to_retrieve { - if !attributes_to_retrieve.contains(attribute) { - continue; - } - } - if !schema.displayed_name().contains(attribute) { - continue; - } - if let Some(pos) = matches_result.get_mut(attribute) { - pos.push(MatchPosition { - start: m.char_index as usize, - length: m.char_length as usize, - }); - } else { - let mut positions = Vec::new(); - positions.push(MatchPosition { - start: m.char_index as usize, - length: m.char_length as usize, - }); - matches_result.insert(attribute.to_string(), positions); - } - } - } - for (_, val) in matches_result.iter_mut() { - val.sort_unstable(); - val.dedup(); - } - matches_result -} - -fn calculate_highlights( - document: &IndexMap, - matches: &MatchesInfos, - attributes_to_highlight: &HashSet, -) -> IndexMap { - let mut highlight_result = document.clone(); - - for (attribute, matches) in matches.iter() { - if attributes_to_highlight.contains(attribute) { - if let Some(Value::String(value)) = document.get(attribute) { - let value: Vec<_> = value.chars().collect(); - let mut highlighted_value = String::new(); - let mut index = 0; - - let longest_matches = matches - .linear_group_by_key(|m| m.start) - .map(|group| group.last().unwrap()) - .filter(move |m| m.start >= index); - - for m in longest_matches { - let before = value.get(index..m.start); - let highlighted = value.get(m.start..(m.start + m.length)); - if let (Some(before), Some(highlighted)) = (before, highlighted) { - highlighted_value.extend(before); - highlighted_value.push_str(""); - highlighted_value.extend(highlighted); - highlighted_value.push_str(""); - index = m.start + m.length; - } else { - error!("value: {:?}; index: {:?}, match: {:?}", value, index, m); - } - } - highlighted_value.extend(value[index..].iter()); - highlight_result.insert(attribute.to_string(), Value::String(highlighted_value)); - }; - } - } - highlight_result -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn aligned_crops() { - let text = r#"En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation."#; - - // simple test - let (start, length) = aligned_crop(&text, 6, 2); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("début", cropped); - - // first word test - let (start, length) = aligned_crop(&text, 0, 1); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("En", cropped); - // last word test - let (start, length) = aligned_crop(&text, 510, 2); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("Fondation", cropped); - - // CJK tests - let text = "this isのス foo myタイリ test"; - - // mixed charset - let (start, length) = aligned_crop(&text, 5, 3); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("isの", cropped); - - // split regular word / CJK word, no space - let (start, length) = aligned_crop(&text, 7, 1); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("の", cropped); - } - - #[test] - fn calculate_matches() { - let mut matches = Vec::new(); - matches.push(Highlight { attribute: 0, char_index: 0, char_length: 3}); - matches.push(Highlight { attribute: 0, char_index: 0, char_length: 2}); - - let mut attributes_to_retrieve: HashSet = HashSet::new(); - attributes_to_retrieve.insert("title".to_string()); - - let schema = Schema::with_primary_key("title"); - - let matches_result = super::calculate_matches(&matches, Some(attributes_to_retrieve), &schema); - - let mut matches_result_expected: HashMap> = HashMap::new(); - - let mut positions = Vec::new(); - positions.push(MatchPosition { - start: 0, - length: 2, - }); - positions.push(MatchPosition { - start: 0, - length: 3, - }); - matches_result_expected.insert("title".to_string(), positions); - - assert_eq!(matches_result, matches_result_expected); - } - - #[test] - fn calculate_highlights() { - let data = r#"{ - "title": "Fondation (Isaac ASIMOV)", - "description": "En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation." - }"#; - - let document: IndexMap = serde_json::from_str(data).unwrap(); - let mut attributes_to_highlight = HashSet::new(); - attributes_to_highlight.insert("title".to_string()); - attributes_to_highlight.insert("description".to_string()); - - let mut matches = HashMap::new(); - - let mut m = Vec::new(); - m.push(MatchPosition { - start: 0, - length: 9, - }); - matches.insert("title".to_string(), m); - - let mut m = Vec::new(); - m.push(MatchPosition { - start: 510, - length: 9, - }); - matches.insert("description".to_string(), m); - let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight); - - let mut result_expected = IndexMap::new(); - result_expected.insert( - "title".to_string(), - Value::String("Fondation (Isaac ASIMOV)".to_string()), - ); - result_expected.insert("description".to_string(), Value::String("En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation.".to_string())); - - assert_eq!(result, result_expected); - } - - #[test] - fn highlight_longest_match() { - let data = r#"{ - "title": "Ice" - }"#; - - let document: IndexMap = serde_json::from_str(data).unwrap(); - let mut attributes_to_highlight = HashSet::new(); - attributes_to_highlight.insert("title".to_string()); - - let mut matches = HashMap::new(); - - let mut m = Vec::new(); - m.push(MatchPosition { - start: 0, - length: 2, - }); - m.push(MatchPosition { - start: 0, - length: 3, - }); - matches.insert("title".to_string(), m); - - let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight); - - let mut result_expected = IndexMap::new(); - result_expected.insert( - "title".to_string(), - Value::String("Ice".to_string()), - ); - - assert_eq!(result, result_expected); - } -} diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index 471336db9..a7ea6c9ee 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -1,5 +1,4 @@ pub mod authentication; -pub mod meilisearch; pub mod normalize_path; pub mod compression; diff --git a/src/models/mod.rs b/src/models/mod.rs deleted file mode 100644 index 82e7e77c4..000000000 --- a/src/models/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod update_operation; diff --git a/src/models/update_operation.rs b/src/models/update_operation.rs deleted file mode 100644 index e7a41b10b..000000000 --- a/src/models/update_operation.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::fmt; - -#[allow(dead_code)] -#[derive(Debug)] -pub enum UpdateOperation { - ClearAllDocuments, - DocumentsAddition, - DocumentsDeletion, - SynonymsUpdate, - SynonymsDeletion, - StopWordsAddition, - StopWordsDeletion, - Schema, - Config, -} - -impl fmt::Display for UpdateOperation { - fn fmt(&self, f: &mut fmt::Formatter) -> std::fmt::Result { - use UpdateOperation::*; - - match self { - ClearAllDocuments => write!(f, "ClearAllDocuments"), - DocumentsAddition => write!(f, "DocumentsAddition"), - DocumentsDeletion => write!(f, "DocumentsDeletion"), - SynonymsUpdate => write!(f, "SynonymsUpdate"), - SynonymsDeletion => write!(f, "SynonymsDelettion"), - StopWordsAddition => write!(f, "StopWordsAddition"), - StopWordsDeletion => write!(f, "StopWordsDeletion"), - Schema => write!(f, "Schema"), - Config => write!(f, "Config"), - } - } -} diff --git a/src/routes/document.rs b/src/routes/document.rs index e8fa0f646..3c4b9acf5 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -38,23 +38,7 @@ async fn get_document( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.main_read_txn()?; - - let internal_id = index - .main - .external_to_internal_docid(&reader, &path.document_id)? - .ok_or(Error::document_not_found(&path.document_id))?; - - let document: Document = index - .document(&reader, None, internal_id)? - .ok_or(Error::document_not_found(&path.document_id))?; - - Ok(HttpResponse::Ok().json(document)) + todo!() } #[delete( @@ -65,17 +49,7 @@ async fn delete_document( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let mut documents_deletion = index.documents_deletion(); - documents_deletion.delete_document_by_external_docid(path.document_id.clone()); - - let update_id = data.db.update_write(|w| documents_deletion.finalize(w))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[derive(Deserialize)] @@ -94,32 +68,7 @@ pub fn get_all_documents_sync( limit: usize, attributes_to_retrieve: Option<&String> ) -> Result, Error> { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - - let documents_ids: Result, _> = index - .documents_fields_counts - .documents_ids(reader)? - .skip(offset) - .take(limit) - .collect(); - - let attributes: Option> = attributes_to_retrieve - .map(|a| a.split(',').collect()); - - let mut documents = Vec::new(); - for document_id in documents_ids? { - if let Ok(Some(document)) = - index.document::(reader, attributes.as_ref(), document_id) - { - documents.push(document); - } - } - - Ok(documents) + todo!() } #[get("/indexes/{index_uid}/documents", wrap = "Authentication::Public")] @@ -128,21 +77,7 @@ async fn get_all_documents( path: web::Path, params: web::Query, ) -> Result { - let offset = params.offset.unwrap_or(0); - let limit = params.limit.unwrap_or(20); - let index_uid = &path.index_uid; - let reader = data.db.main_read_txn()?; - - let documents = get_all_documents_sync( - &data, - &reader, - index_uid, - offset, - limit, - params.attributes_to_retrieve.as_ref() - )?; - - Ok(HttpResponse::Ok().json(documents)) + todo!() } fn find_primary_key(document: &IndexMap) -> Option { @@ -167,41 +102,7 @@ async fn update_multiple_documents( body: web::Json>, is_partial: bool, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let reader = data.db.main_read_txn()?; - - let mut schema = index - .main - .schema(&reader)? - .ok_or(meilisearch_core::Error::SchemaMissing)?; - - if schema.primary_key().is_none() { - let id = match ¶ms.primary_key { - Some(id) => id.to_string(), - None => body - .first() - .and_then(find_primary_key) - .ok_or(meilisearch_core::Error::MissingPrimaryKey)?, - }; - - schema.set_primary_key(&id).map_err(Error::bad_request)?; - - data.db.main_write(|w| index.main.put_schema(w, &schema))?; - } - - let mut document_addition = if is_partial { - index.documents_partial_addition() - } else { - index.documents_addition() - }; - - for document in body.into_inner() { - document_addition.update_document(document); - } - - Ok(data.db.update_write(|w| document_addition.finalize(w))?) - })?; - return Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))); + todo!() } #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] @@ -211,7 +112,7 @@ async fn add_documents( params: web::Query, body: web::Json>, ) -> Result { - update_multiple_documents(data, path, params, body, false).await + todo!() } #[put("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] @@ -233,21 +134,7 @@ async fn delete_documents( path: web::Path, body: web::Json>, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let mut documents_deletion = index.documents_deletion(); - - for document_id in body.into_inner() { - let document_id = update::value_to_string(&document_id); - documents_deletion.delete_document_by_external_docid(document_id); - } - - let update_id = data.db.update_write(|w| documents_deletion.finalize(w))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] @@ -255,12 +142,5 @@ async fn clear_all_documents( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let update_id = data.db.update_write(|w| index.clear_all(w))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } diff --git a/src/routes/dump.rs b/src/routes/dump.rs index 97fafdfa8..c46b0e502 100644 --- a/src/routes/dump.rs +++ b/src/routes/dump.rs @@ -19,11 +19,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { async fn trigger_dump( data: web::Data, ) -> Result { - let dumps_dir = Path::new(&data.dumps_dir); - match init_dump_process(&data, &dumps_dir) { - Ok(resume) => Ok(HttpResponse::Accepted().json(resume)), - Err(e) => Err(e.into()) - } + todo!() } #[derive(Debug, Serialize)] @@ -42,23 +38,5 @@ async fn get_dump_status( data: web::Data, path: web::Path, ) -> Result { - let dumps_dir = Path::new(&data.dumps_dir); - let dump_uid = &path.dump_uid; - - if let Some(resume) = DumpInfo::get_current() { - if &resume.uid == dump_uid { - return Ok(HttpResponse::Ok().json(resume)); - } - } - - if File::open(compressed_dumps_dir(Path::new(dumps_dir), dump_uid)).is_ok() { - let resume = DumpInfo::new( - dump_uid.into(), - DumpStatus::Done - ); - - Ok(HttpResponse::Ok().json(resume)) - } else { - Err(Error::not_found("dump does not exist").into()) - } + todo!() } diff --git a/src/routes/index.rs b/src/routes/index.rs index aa0496920..9a00030d0 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -23,12 +23,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { } fn generate_uid() -> String { - let mut rng = rand::thread_rng(); - let sample = b"abcdefghijklmnopqrstuvwxyz0123456789"; - sample - .choose_multiple(&mut rng, 8) - .map(|c| *c as char) - .collect() + todo!() } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -42,54 +37,7 @@ pub struct IndexResponse { } pub fn list_indexes_sync(data: &web::Data, reader: &MainReader) -> Result, ResponseError> { - let mut indexes = Vec::new(); - - for index_uid in data.db.indexes_uids() { - let index = data.db.open_index(&index_uid); - - match index { - Some(index) => { - let name = index.main.name(reader)?.ok_or(Error::internal( - "Impossible to get the name of an index", - ))?; - let created_at = index - .main - .created_at(reader)? - .ok_or(Error::internal( - "Impossible to get the create date of an index", - ))?; - let updated_at = index - .main - .updated_at(reader)? - .ok_or(Error::internal( - "Impossible to get the last update date of an index", - ))?; - - let primary_key = match index.main.schema(reader) { - Ok(Some(schema)) => match schema.primary_key() { - Some(primary_key) => Some(primary_key.to_owned()), - None => None, - }, - _ => None, - }; - - let index_response = IndexResponse { - name, - uid: index_uid, - created_at, - updated_at, - primary_key, - }; - indexes.push(index_response); - } - None => error!( - "Index {} is referenced in the indexes list but cannot be found", - index_uid - ), - } - } - - Ok(indexes) + todo!() } #[get("/indexes", wrap = "Authentication::Private")] @@ -105,44 +53,7 @@ async fn get_index( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.main_read_txn()?; - let name = index.main.name(&reader)?.ok_or(Error::internal( - "Impossible to get the name of an index", - ))?; - let created_at = index - .main - .created_at(&reader)? - .ok_or(Error::internal( - "Impossible to get the create date of an index", - ))?; - let updated_at = index - .main - .updated_at(&reader)? - .ok_or(Error::internal( - "Impossible to get the last update date of an index", - ))?; - - let primary_key = match index.main.schema(&reader) { - Ok(Some(schema)) => match schema.primary_key() { - Some(primary_key) => Some(primary_key.to_owned()), - None => None, - }, - _ => None, - }; - let index_response = IndexResponse { - name, - uid: path.index_uid.clone(), - created_at, - updated_at, - primary_key, - }; - - Ok(HttpResponse::Ok().json(index_response)) + todo!() } #[derive(Debug, Deserialize)] @@ -160,46 +71,7 @@ pub fn create_index_sync( name: String, primary_key: Option, ) -> Result { - - let created_index = database - .create_index(&uid) - .map_err(|e| match e { - meilisearch_core::Error::IndexAlreadyExists => Error::IndexAlreadyExists(uid.clone()), - _ => Error::create_index(e) - })?; - - let index_response = database.main_write::<_, _, Error>(|mut write_txn| { - created_index.main.put_name(&mut write_txn, &name)?; - - let created_at = created_index - .main - .created_at(&write_txn)? - .ok_or(Error::internal("Impossible to read created at"))?; - - let updated_at = created_index - .main - .updated_at(&write_txn)? - .ok_or(Error::internal("Impossible to read updated at"))?; - - if let Some(id) = primary_key.clone() { - if let Some(mut schema) = created_index.main.schema(&write_txn)? { - schema - .set_primary_key(&id) - .map_err(Error::bad_request)?; - created_index.main.put_schema(&mut write_txn, &schema)?; - } - } - let index_response = IndexResponse { - name, - uid, - created_at, - updated_at, - primary_key, - }; - Ok(index_response) - })?; - - Ok(index_response) + todo!() } #[post("/indexes", wrap = "Authentication::Private")] @@ -207,36 +79,7 @@ async fn create_index( data: web::Data, body: web::Json, ) -> Result { - if let (None, None) = (body.name.clone(), body.uid.clone()) { - return Err(Error::bad_request( - "Index creation must have an uid", - ).into()); - } - - let uid = match &body.uid { - Some(uid) => { - if uid - .chars() - .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') - { - uid.to_owned() - } else { - return Err(Error::InvalidIndexUid.into()); - } - } - None => loop { - let uid = generate_uid(); - if data.db.open_index(&uid).is_none() { - break uid; - } - }, - }; - - let name = body.name.as_ref().unwrap_or(&uid).to_string(); - - let index_response = create_index_sync(&data.db, uid, name, body.primary_key.clone())?; - - Ok(HttpResponse::Created().json(index_response)) + todo!() } #[derive(Debug, Deserialize)] @@ -262,60 +105,7 @@ async fn update_index( path: web::Path, body: web::Json, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - data.db.main_write::<_, _, ResponseError>(|writer| { - if let Some(name) = &body.name { - index.main.put_name(writer, name)?; - } - - if let Some(id) = body.primary_key.clone() { - if let Some(mut schema) = index.main.schema(writer)? { - schema.set_primary_key(&id)?; - index.main.put_schema(writer, &schema)?; - } - } - index.main.put_updated_at(writer)?; - Ok(()) - })?; - - let reader = data.db.main_read_txn()?; - let name = index.main.name(&reader)?.ok_or(Error::internal( - "Impossible to get the name of an index", - ))?; - let created_at = index - .main - .created_at(&reader)? - .ok_or(Error::internal( - "Impossible to get the create date of an index", - ))?; - let updated_at = index - .main - .updated_at(&reader)? - .ok_or(Error::internal( - "Impossible to get the last update date of an index", - ))?; - - let primary_key = match index.main.schema(&reader) { - Ok(Some(schema)) => match schema.primary_key() { - Some(primary_key) => Some(primary_key.to_owned()), - None => None, - }, - _ => None, - }; - - let index_response = IndexResponse { - name, - uid: path.index_uid.clone(), - created_at, - updated_at, - primary_key, - }; - - Ok(HttpResponse::Ok().json(index_response)) + todo!() } #[delete("/indexes/{index_uid}", wrap = "Authentication::Private")] @@ -323,11 +113,7 @@ async fn delete_index( data: web::Data, path: web::Path, ) -> Result { - if data.db.delete_index(&path.index_uid)? { - Ok(HttpResponse::NoContent().finish()) - } else { - Err(Error::index_not_found(&path.index_uid).into()) - } + todo!() } #[derive(Deserialize)] @@ -344,34 +130,14 @@ async fn get_update_status( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.update_read_txn()?; - - let status = index.update_status(&reader, path.update_id)?; - - match status { - Some(status) => Ok(HttpResponse::Ok().json(status)), - None => Err(Error::NotFound(format!( - "Update {}", - path.update_id - )).into()), - } + todo!() } pub fn get_all_updates_status_sync( data: &web::Data, reader: &UpdateReader, index_uid: &str, ) -> Result, Error> { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - Ok(index.all_updates_status(reader)?) + todo!() } #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] @@ -379,10 +145,5 @@ async fn get_all_updates_status( data: web::Data, path: web::Path, ) -> Result { - - let reader = data.db.update_read_txn()?; - - let response = get_all_updates_status_sync(&data, &reader, &path.index_uid)?; - - Ok(HttpResponse::Ok().json(response)) + todo!() } diff --git a/src/routes/key.rs b/src/routes/key.rs index a0cbaccc3..f4f313e4e 100644 --- a/src/routes/key.rs +++ b/src/routes/key.rs @@ -18,9 +18,5 @@ struct KeysResponse { #[get("/keys", wrap = "Authentication::Admin")] async fn list(data: web::Data) -> HttpResponse { - let api_keys = data.api_keys.clone(); - HttpResponse::Ok().json(KeysResponse { - private: api_keys.private, - public: api_keys.public, - }) + todo!() } diff --git a/src/routes/search.rs b/src/routes/search.rs index 3cd3c3f60..8d8f750a9 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -40,8 +40,7 @@ async fn search_with_url_query( path: web::Path, params: web::Query, ) -> Result { - let search_result = params.search(&path.index_uid, data)?; - Ok(HttpResponse::Ok().json(search_result)) + todo!() } #[derive(Deserialize)] @@ -95,175 +94,5 @@ impl SearchQuery { index_uid: &str, data: web::Data, ) -> Result { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - let reader = data.db.main_read_txn()?; - let schema = index - .main - .schema(&reader)? - .ok_or(Error::internal("Impossible to retrieve the schema"))?; - - let query = self - .q - .clone() - .and_then(|q| if q.is_empty() { None } else { Some(q) }); - - let mut search_builder = index.new_search(query); - - if let Some(offset) = self.offset { - search_builder.offset(offset); - } - if let Some(limit) = self.limit { - search_builder.limit(limit); - } - - let available_attributes = schema.displayed_name(); - let mut restricted_attributes: HashSet<&str>; - match &self.attributes_to_retrieve { - Some(attributes_to_retrieve) => { - let attributes_to_retrieve: HashSet<&str> = - attributes_to_retrieve.split(',').collect(); - if attributes_to_retrieve.contains("*") { - restricted_attributes = available_attributes.clone(); - } else { - restricted_attributes = HashSet::new(); - for attr in attributes_to_retrieve { - if available_attributes.contains(attr) { - restricted_attributes.insert(attr); - search_builder.add_retrievable_field(attr.to_string()); - } else { - warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr); - } - } - } - } - None => { - restricted_attributes = available_attributes.clone(); - } - } - - if let Some(ref facet_filters) = self.facet_filters { - let attrs = index - .main - .attributes_for_faceting(&reader)? - .unwrap_or_default(); - search_builder.add_facet_filters(FacetFilter::from_str( - facet_filters, - &schema, - &attrs, - )?); - } - - if let Some(facets) = &self.facets_distribution { - match index.main.attributes_for_faceting(&reader)? { - Some(ref attrs) => { - let field_ids = prepare_facet_list(&facets, &schema, attrs)?; - search_builder.add_facets(field_ids); - } - None => return Err(FacetCountError::NoFacetSet.into()), - } - } - - if let Some(attributes_to_crop) = &self.attributes_to_crop { - let default_length = self.crop_length.unwrap_or(200); - let mut final_attributes: HashMap = HashMap::new(); - - for attribute in attributes_to_crop.split(',') { - let mut attribute = attribute.split(':'); - let attr = attribute.next(); - let length = attribute - .next() - .and_then(|s| s.parse().ok()) - .unwrap_or(default_length); - match attr { - Some("*") => { - for attr in &restricted_attributes { - final_attributes.insert(attr.to_string(), length); - } - } - Some(attr) => { - if available_attributes.contains(attr) { - final_attributes.insert(attr.to_string(), length); - } else { - warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr); - } - } - None => (), - } - } - search_builder.attributes_to_crop(final_attributes); - } - - if let Some(attributes_to_highlight) = &self.attributes_to_highlight { - let mut final_attributes: HashSet = HashSet::new(); - for attribute in attributes_to_highlight.split(',') { - if attribute == "*" { - for attr in &restricted_attributes { - final_attributes.insert(attr.to_string()); - } - } else if available_attributes.contains(attribute) { - final_attributes.insert(attribute.to_string()); - } else { - warn!("The attributes {:?} present in attributesToHighlight parameter doesn't exist", attribute); - } - } - - search_builder.attributes_to_highlight(final_attributes); - } - - if let Some(filters) = &self.filters { - search_builder.filters(filters.to_string()); - } - - if let Some(matches) = self.matches { - if matches { - search_builder.get_matches(); - } - } - search_builder.search(&reader) - } -} - -/// Parses the incoming string into an array of attributes for which to return a count. It returns -/// a Vec of attribute names ascociated with their id. -/// -/// An error is returned if the array is malformed, or if it contains attributes that are -/// unexisting, or not set as facets. -fn prepare_facet_list( - facets: &str, - schema: &Schema, - facet_attrs: &[FieldId], -) -> Result, FacetCountError> { - let json_array = serde_json::from_str(facets)?; - match json_array { - Value::Array(vals) => { - let wildcard = Value::String("*".to_string()); - if vals.iter().any(|f| f == &wildcard) { - let attrs = facet_attrs - .iter() - .filter_map(|&id| schema.name(id).map(|n| (id, n.to_string()))) - .collect(); - return Ok(attrs); - } - let mut field_ids = Vec::with_capacity(facet_attrs.len()); - for facet in vals { - match facet { - Value::String(facet) => { - if let Some(id) = schema.id(&facet) { - if !facet_attrs.contains(&id) { - return Err(FacetCountError::AttributeNotSet(facet)); - } - field_ids.push((id, facet)); - } - } - bad_val => return Err(FacetCountError::unexpected_token(bad_val, &["String"])), - } - } - Ok(field_ids) - } - bad_val => Err(FacetCountError::unexpected_token(bad_val, &["[String]"])), - } + todo!() } diff --git a/src/routes/setting.rs b/src/routes/setting.rs index 00562eed0..36c7fd239 100644 --- a/src/routes/setting.rs +++ b/src/routes/setting.rs @@ -32,91 +32,13 @@ pub fn services(cfg: &mut web::ServiceConfig) { .service(update_attributes_for_faceting); } -pub fn update_all_settings_txn( - data: &web::Data, - settings: SettingsUpdate, - index_uid: &str, - write_txn: &mut UpdateWriter, -) -> Result { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - let update_id = index.settings_update(write_txn, settings)?; - Ok(update_id) -} - #[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] async fn update_all( data: web::Data, path: web::Path, body: web::Json, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - Ok(data.db.update_write::<_, _, ResponseError>(|writer| { - let settings = body.into_inner().to_update().map_err(Error::bad_request)?; - let update_id = index.settings_update(writer, settings)?; - Ok(update_id) - })?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -pub fn get_all_sync(data: &web::Data, reader: &MainReader, index_uid: &str) -> Result { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - let stop_words: BTreeSet = index.main.stop_words(&reader)?.into_iter().collect(); - - let synonyms_list = index.main.synonyms(reader)?; - - let mut synonyms = BTreeMap::new(); - let index_synonyms = &index.synonyms; - for synonym in synonyms_list { - let list = index_synonyms.synonyms(reader, synonym.as_bytes())?; - synonyms.insert(synonym, list); - } - - let ranking_rules = index - .main - .ranking_rules(reader)? - .unwrap_or(DEFAULT_RANKING_RULES.to_vec()) - .into_iter() - .map(|r| r.to_string()) - .collect(); - - let schema = index.main.schema(&reader)?; - - let distinct_attribute = match (index.main.distinct_attribute(reader)?, &schema) { - (Some(id), Some(schema)) => schema.name(id).map(str::to_string), - _ => None, - }; - - let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(&reader)?) { - (Some(schema), Some(attrs)) => attrs - .iter() - .filter_map(|&id| schema.name(id)) - .map(str::to_string) - .collect(), - _ => vec![], - }; - - let searchable_attributes = schema.as_ref().map(get_indexed_attributes); - let displayed_attributes = schema.as_ref().map(get_displayed_attributes); - - Ok(Settings { - ranking_rules: Some(Some(ranking_rules)), - distinct_attribute: Some(distinct_attribute), - searchable_attributes: Some(searchable_attributes), - displayed_attributes: Some(displayed_attributes), - stop_words: Some(Some(stop_words)), - synonyms: Some(Some(synonyms)), - attributes_for_faceting: Some(Some(attributes_for_faceting)), - }) + todo!() } #[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] @@ -124,10 +46,7 @@ async fn get_all( data: web::Data, path: web::Path, ) -> Result { - let reader = data.db.main_read_txn()?; - let settings = get_all_sync(&data, &reader, &path.index_uid)?; - - Ok(HttpResponse::Ok().json(settings)) + todo!() } #[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] @@ -135,27 +54,7 @@ async fn delete_all( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - ranking_rules: UpdateState::Clear, - distinct_attribute: UpdateState::Clear, - primary_key: UpdateState::Clear, - searchable_attributes: UpdateState::Clear, - displayed_attributes: UpdateState::Clear, - stop_words: UpdateState::Clear, - synonyms: UpdateState::Clear, - attributes_for_faceting: UpdateState::Clear, - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[get( @@ -166,21 +65,7 @@ async fn get_rules( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - - let ranking_rules = index - .main - .ranking_rules(&reader)? - .unwrap_or(DEFAULT_RANKING_RULES.to_vec()) - .into_iter() - .map(|r| r.to_string()) - .collect::>(); - - Ok(HttpResponse::Ok().json(ranking_rules)) + todo!() } #[post( @@ -192,19 +77,7 @@ async fn update_rules( path: web::Path, body: web::Json>>, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - ranking_rules: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[delete( @@ -215,21 +88,7 @@ async fn delete_rules( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - ranking_rules: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[get( @@ -240,19 +99,7 @@ async fn get_distinct( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - let distinct_attribute_id = index.main.distinct_attribute(&reader)?; - let schema = index.main.schema(&reader)?; - let distinct_attribute = match (schema, distinct_attribute_id) { - (Some(schema), Some(id)) => schema.name(id).map(str::to_string), - _ => None, - }; - - Ok(HttpResponse::Ok().json(distinct_attribute)) + todo!() } #[post( @@ -264,19 +111,7 @@ async fn update_distinct( path: web::Path, body: web::Json>, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - distinct_attribute: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[delete( @@ -287,21 +122,7 @@ async fn delete_distinct( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - distinct_attribute: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[get( @@ -312,15 +133,7 @@ async fn get_searchable( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - let schema = index.main.schema(&reader)?; - let searchable_attributes: Option> = schema.as_ref().map(get_indexed_attributes); - - Ok(HttpResponse::Ok().json(searchable_attributes)) + todo!() } #[post( @@ -332,20 +145,7 @@ async fn update_searchable( path: web::Path, body: web::Json>>, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - searchable_attributes: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[delete( @@ -356,21 +156,7 @@ async fn delete_searchable( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - searchable_attributes: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[get( @@ -381,17 +167,7 @@ async fn get_displayed( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - - let schema = index.main.schema(&reader)?; - - let displayed_attributes = schema.as_ref().map(get_displayed_attributes); - - Ok(HttpResponse::Ok().json(displayed_attributes)) + todo!() } #[post( @@ -403,19 +179,7 @@ async fn update_displayed( path: web::Path, body: web::Json>>, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - displayed_attributes: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[delete( @@ -426,21 +190,7 @@ async fn delete_displayed( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - displayed_attributes: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[get( @@ -451,26 +201,7 @@ async fn get_attributes_for_faceting( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let attributes_for_faceting = data.db.main_read::<_, _, ResponseError>(|reader| { - let schema = index.main.schema(reader)?; - let attrs = index.main.attributes_for_faceting(reader)?; - let attr_names = match (&schema, &attrs) { - (Some(schema), Some(attrs)) => attrs - .iter() - .filter_map(|&id| schema.name(id)) - .map(str::to_string) - .collect(), - _ => vec![], - }; - Ok(attr_names) - })?; - - Ok(HttpResponse::Ok().json(attributes_for_faceting)) + todo!() } #[post( @@ -482,19 +213,7 @@ async fn update_attributes_for_faceting( path: web::Path, body: web::Json>>, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - attributes_for_faceting: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[delete( @@ -505,43 +224,5 @@ async fn delete_attributes_for_faceting( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - attributes_for_faceting: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -fn get_indexed_attributes(schema: &Schema) -> Vec { - if schema.is_indexed_all() { - ["*"].iter().map(|s| s.to_string()).collect() - } else { - schema - .indexed_name() - .iter() - .map(|s| s.to_string()) - .collect() - } -} - -fn get_displayed_attributes(schema: &Schema) -> BTreeSet { - if schema.is_displayed_all() { - ["*"].iter().map(|s| s.to_string()).collect() - } else { - schema - .displayed_name() - .iter() - .map(|s| s.to_string()) - .collect() - } + todo!() } diff --git a/src/routes/stats.rs b/src/routes/stats.rs index f8c531732..734811450 100644 --- a/src/routes/stats.rs +++ b/src/routes/stats.rs @@ -32,30 +32,7 @@ async fn index_stats( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.main_read_txn()?; - - let number_of_documents = index.main.number_of_documents(&reader)?; - - let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default(); - - let update_reader = data.db.update_read_txn()?; - - let is_indexing = - data.db.is_indexing(&update_reader, &path.index_uid)? - .ok_or(Error::internal( - "Impossible to know if the database is indexing", - ))?; - - Ok(HttpResponse::Ok().json(IndexStatsResponse { - number_of_documents, - is_indexing, - fields_distribution, - })) + todo!() } #[derive(Serialize)] @@ -68,52 +45,7 @@ struct StatsResult { #[get("/stats", wrap = "Authentication::Private")] async fn get_stats(data: web::Data) -> Result { - let mut index_list = HashMap::new(); - - let reader = data.db.main_read_txn()?; - let update_reader = data.db.update_read_txn()?; - - let indexes_set = data.db.indexes_uids(); - for index_uid in indexes_set { - let index = data.db.open_index(&index_uid); - match index { - Some(index) => { - let number_of_documents = index.main.number_of_documents(&reader)?; - - let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default(); - - let is_indexing = data.db.is_indexing(&update_reader, &index_uid)?.ok_or( - Error::internal("Impossible to know if the database is indexing"), - )?; - - let response = IndexStatsResponse { - number_of_documents, - is_indexing, - fields_distribution, - }; - index_list.insert(index_uid, response); - } - None => error!( - "Index {:?} is referenced in the indexes list but cannot be found", - index_uid - ), - } - } - - let database_size = WalkDir::new(&data.db_path) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len()); - - let last_update = data.db.last_update(&reader)?; - - Ok(HttpResponse::Ok().json(StatsResult { - database_size, - last_update, - indexes: index_list, - })) + todo!() } #[derive(Serialize)] @@ -126,9 +58,5 @@ struct VersionResponse { #[get("/version", wrap = "Authentication::Private")] async fn get_version() -> HttpResponse { - HttpResponse::Ok().json(VersionResponse { - commit_sha: env!("VERGEN_SHA").to_string(), - build_date: env!("VERGEN_BUILD_TIMESTAMP").to_string(), - pkg_version: env!("CARGO_PKG_VERSION").to_string(), - }) + todo!() } diff --git a/src/routes/stop_words.rs b/src/routes/stop_words.rs index c757b4d14..f57d9cdd7 100644 --- a/src/routes/stop_words.rs +++ b/src/routes/stop_words.rs @@ -20,14 +20,7 @@ async fn get( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - let stop_words = index.main.stop_words(&reader)?; - - Ok(HttpResponse::Ok().json(stop_words)) + todo!() } #[post( @@ -39,18 +32,7 @@ async fn update( path: web::Path, body: web::Json>, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = SettingsUpdate { - stop_words: UpdateState::Update(body.into_inner()), - ..SettingsUpdate::default() - }; - - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[delete( @@ -61,19 +43,5 @@ async fn delete( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - stop_words: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } diff --git a/src/routes/synonym.rs b/src/routes/synonym.rs index 5aefaaca5..398889473 100644 --- a/src/routes/synonym.rs +++ b/src/routes/synonym.rs @@ -22,23 +22,7 @@ async fn get( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.main_read_txn()?; - - let synonyms_list = index.main.synonyms(&reader)?; - - let mut synonyms = IndexMap::new(); - let index_synonyms = &index.synonyms; - for synonym in synonyms_list { - let list = index_synonyms.synonyms(&reader, synonym.as_bytes())?; - synonyms.insert(synonym, list); - } - - Ok(HttpResponse::Ok().json(synonyms)) + todo!() } #[post( @@ -50,18 +34,7 @@ async fn update( path: web::Path, body: web::Json>>, ) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = SettingsUpdate { - synonyms: UpdateState::Update(body.into_inner()), - ..SettingsUpdate::default() - }; - - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } #[delete( @@ -72,19 +45,5 @@ async fn delete( data: web::Data, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - synonyms: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + todo!() } From 7c9eaaeadb8a6a0c7af4d34f62732dfcea3dacb9 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 22 Dec 2020 14:02:41 +0100 Subject: [PATCH 003/522] clean code, and fix errors --- .gitignore | 2 + Cargo.lock | 3447 ++++++++++++++++++++++++++++++++++++++ Cargo.toml | 54 +- src/data.rs | 109 +- src/dump.rs | 4 +- src/error.rs | 25 +- src/lib.rs | 55 +- src/main.rs | 41 +- src/option.rs | 15 +- src/routes/document.rs | 88 +- src/routes/index.rs | 68 +- src/routes/key.rs | 2 +- src/routes/mod.rs | 4 +- src/routes/search.rs | 72 +- src/routes/setting.rs | 94 +- src/routes/stats.rs | 10 +- src/routes/stop_words.rs | 19 +- src/routes/synonym.rs | 20 +- src/updates/mod.rs | 17 + src/updates/settings.rs | 51 + 20 files changed, 3723 insertions(+), 474 deletions(-) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 src/updates/mod.rs create mode 100644 src/updates/settings.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..1d71f78ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# the milli project is a library +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..0b7386594 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3447 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "actix-codec" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" +dependencies = [ + "bitflags", + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project 0.4.27", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-connect" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "derive_more", + "either", + "futures-util", + "http", + "log", + "rustls 0.18.1", + "tokio-rustls", + "trust-dns-proto", + "trust-dns-resolver", + "webpki", +] + +[[package]] +name = "actix-cors" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3a3d5493dbc9b8769fe88c030d057ef8d2edc5728e5e26267780e8fc5db0be" +dependencies = [ + "actix-web", + "derive_more", + "futures-util", + "log", + "once_cell", + "tinyvec", +] + +[[package]] +name = "actix-http" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" +dependencies = [ + "actix-codec", + "actix-connect", + "actix-rt", + "actix-service", + "actix-threadpool", + "actix-tls", + "actix-utils", + "base64 0.13.0", + "bitflags", + "brotli2", + "bytes 0.5.6", + "cookie", + "copyless", + "derive_more", + "either", + "encoding_rs", + "flate2", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "h2", + "http", + "httparse", + "indexmap", + "itoa", + "language-tags", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project 1.0.2", + "rand 0.7.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1 0.9.2", + "slab", + "time 0.2.23", +] + +[[package]] +name = "actix-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd1f7dbda1645bf7da33554db60891755f6c01c1b2169e2f4c492098d30c235" +dependencies = [ + "bytestring", + "http", + "log", + "regex", + "serde", +] + +[[package]] +name = "actix-rt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" +dependencies = [ + "actix-macros", + "actix-threadpool", + "copyless", + "futures-channel", + "futures-util", + "smallvec", + "tokio", +] + +[[package]] +name = "actix-server" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "futures-channel", + "futures-util", + "log", + "mio", + "mio-uds", + "num_cpus", + "slab", + "socket2", +] + +[[package]] +name = "actix-service" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" +dependencies = [ + "futures-util", + "pin-project 0.4.27", +] + +[[package]] +name = "actix-testing" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" +dependencies = [ + "actix-macros", + "actix-rt", + "actix-server", + "actix-service", + "log", + "socket2", +] + +[[package]] +name = "actix-threadpool" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" +dependencies = [ + "derive_more", + "futures-channel", + "lazy_static", + "log", + "num_cpus", + "parking_lot", + "threadpool", +] + +[[package]] +name = "actix-tls" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24789b7d7361cf5503a504ebe1c10806896f61e96eca9a7350e23001aca715fb" +dependencies = [ + "actix-codec", + "actix-service", + "actix-utils", + "futures-util", + "rustls 0.18.1", + "tokio-rustls", + "webpki", + "webpki-roots 0.20.0", +] + +[[package]] +name = "actix-utils" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "bitflags", + "bytes 0.5.6", + "either", + "futures-channel", + "futures-sink", + "futures-util", + "log", + "pin-project 0.4.27", + "slab", +] + +[[package]] +name = "actix-web" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-testing", + "actix-threadpool", + "actix-tls", + "actix-utils", + "actix-web-codegen", + "awc", + "bytes 0.5.6", + "derive_more", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "log", + "mime", + "pin-project 1.0.2", + "regex", + "rustls 0.18.1", + "serde", + "serde_json", + "serde_urlencoded", + "socket2", + "time 0.2.23", + "tinyvec", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad26f77093333e0e7c6ffe54ebe3582d908a104e448723eec6d43d08b07143fb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "addr2line" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "anyhow" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479" + +[[package]] +name = "assert-json-diff" +version = "1.0.1" +source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-trait" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "awc" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" +dependencies = [ + "actix-codec", + "actix-http", + "actix-rt", + "actix-service", + "base64 0.13.0", + "bytes 0.5.6", + "cfg-if 1.0.0", + "derive_more", + "futures-core", + "log", + "mime", + "percent-encoding", + "rand 0.7.3", + "rustls 0.18.1", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "backtrace" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +dependencies = [ + "addr2line", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bincode" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" +dependencies = [ + "byteorder", + "serde", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "bstr" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byte-unit" +version = "4.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c8758c32833faaae35b24a73d332e62d0528e89076ae841c63940e37008b153" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" + +[[package]] +name = "bytestring" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7c05fa5172da78a62d9949d662d2ac89d4cc7355d7b49adee5163f1fb3f363" +dependencies = [ + "bytes 0.5.6", +] + +[[package]] +name = "cc" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "chunked_transfer" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca" + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "term_size", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "const_fn" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" + +[[package]] +name = "cookie" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +dependencies = [ + "percent-encoding", + "time 0.2.23", + "version_check", +] + +[[package]] +name = "copyless" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.1", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils 0.8.1", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils 0.8.1", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "csv" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "debugid" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba" +dependencies = [ + "serde", + "uuid", +] + +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +dependencies = [ + "atty", + "humantime 2.0.1", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "filetime" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "winapi 0.3.9", +] + +[[package]] +name = "flate2" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + +[[package]] +name = "fst" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" + +[[package]] +name = "futures-executor" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" + +[[package]] +name = "futures-macro" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" + +[[package]] +name = "futures-task" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project 1.0.2", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "grenad" +version = "0.1.0" +source = "git+https://github.com/Kerollmops/grenad.git?rev=3adcb26#3adcb267dcbc590c7da10eb5f887a254865b3dbe" +dependencies = [ + "byteorder", + "flate2", + "log", + "nix", + "snap", + "tempfile", + "zstd", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heed" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" +dependencies = [ + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-rkv-sys", + "once_cell", + "page_size", + "synchronoise", + "url", + "zerocopy", +] + +[[package]] +name = "heed-traits" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" + +[[package]] +name = "heed-types" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" +dependencies = [ + "bincode", + "heed-traits", + "serde", + "serde_json", + "zerocopy", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + +[[package]] +name = "http" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84129d298a6d57d246960ff8eb831ca4af3f96d29e2e28848dae275408658e26" +dependencies = [ + "bytes 0.5.6", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "human_format" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "humantime" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" + +[[package]] +name = "hyper" +version = "0.13.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.2", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" +dependencies = [ + "bytes 0.5.6", + "futures-util", + "hyper", + "log", + "rustls 0.18.1", + "tokio", + "tokio-rustls", + "webpki", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "im" +version = "14.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696059c87b83c5a258817ecd67c3af915e3ed141891fc35a1e79908801cf0ce7" +dependencies = [ + "bitmaps", + "rand_core 0.5.1", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "indexmap" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +dependencies = [ + "autocfg", + "hashbrown", + "serde", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "jemalloc-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", +] + +[[package]] +name = "jobserver" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "levenshtein_automata" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a" +dependencies = [ + "fst", +] + +[[package]] +name = "libc" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" + +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + +[[package]] +name = "lmdb-rkv-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b27470ac25167b3afdfb6af8fcd3bc1be67de50ffbdaf4073378cfded6ae24a5" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "main_error" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb63bb1e282e0b6aba0addb1f0e87cb5181ea68142b2dfd21ba108f8e8088a64" + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "meilisearch-error" +version = "0.15.0" +dependencies = [ + "actix-http", +] + +[[package]] +name = "meilisearch-http" +version = "0.17.0" +dependencies = [ + "actix-cors", + "actix-http", + "actix-rt", + "actix-service", + "actix-web", + "assert-json-diff", + "byte-unit", + "bytes 0.6.0", + "chrono", + "crossbeam-channel", + "env_logger 0.8.2", + "flate2", + "futures", + "http", + "indexmap", + "jemallocator", + "log", + "main_error", + "meilisearch-error", + "milli", + "mime", + "once_cell", + "rand 0.7.3", + "regex", + "rustls 0.18.1", + "sentry", + "serde", + "serde_json", + "serde_qs", + "serde_url_params", + "sha2", + "siphasher", + "slice-group-by", + "structopt", + "tar", + "tempdir", + "tempfile", + "tokio", + "ureq", + "vergen", + "walkdir", + "whoami", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + +[[package]] +name = "milli" +version = "0.1.0" +dependencies = [ + "anyhow", + "bstr", + "byte-unit", + "byteorder", + "crossbeam-channel", + "csv", + "either", + "flate2", + "fst", + "fxhash", + "grenad", + "heed", + "human_format", + "itertools", + "jemallocator", + "levenshtein_automata", + "linked-hash-map", + "log", + "memmap", + "near-proximity", + "num-traits", + "obkv", + "once_cell", + "ordered-float", + "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", + "pest_derive", + "rayon", + "regex", + "ringtail", + "roaring", + "serde", + "serde_json", + "slice-group-by", + "smallstr", + "smallvec", + "stderrlog", + "structopt", + "tempfile", + "uuid", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "near-proximity" +version = "0.1.0" +source = "git+https://github.com/Kerollmops/plane-sweep-proximity?rev=6608205#66082058537f6fe7709adc4690048d62f3c0e9b7" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nix" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" + +[[package]] +name = "obkv" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" + +[[package]] +name = "once_cell" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "ordered-float" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dacdec97876ef3ede8c50efc429220641a0b11ba0048b4b0c357bccbc47c5204" +dependencies = [ + "num-traits", +] + +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest" +version = "2.1.3" +source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sha-1 0.8.2", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +dependencies = [ + "pin-project-internal 1.0.2", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-project-lite" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "qstring" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d464fae65fff2680baf48019211ce37aaec0c78e9264c84a3e484717f965104e" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils 0.8.1", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "regex" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", +] + +[[package]] +name = "regex-syntax" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" +dependencies = [ + "base64 0.13.0", + "bytes 0.5.6", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite 0.2.0", + "rustls 0.18.1", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-rustls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.20.0", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "ring" +version = "0.16.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "ringtail" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21215c1b9d8f7832b433255bd9eea3e2779aa55b21b2f8e13aad62c74749b237" + +[[package]] +name = "roaring" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb550891a98438463978260676feef06c12bfb0eb0b05e191f888fb785cc9374" +dependencies = [ + "byteorder", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64 0.12.3", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "sentry" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b01b723fc1b0a0f9394ca1a8451daec6e20206d47f96c3dceea7fd11ec9eec0" +dependencies = [ + "backtrace", + "env_logger 0.7.1", + "failure", + "hostname", + "httpdate", + "im", + "lazy_static", + "libc", + "log", + "rand 0.7.3", + "regex", + "reqwest", + "rustc_version", + "sentry-types", + "uname", + "url", +] + +[[package]] +name = "sentry-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ec406c11c060c8a7d5d67fc6f4beb2888338dcb12b9af409451995f124749d" +dependencies = [ + "chrono", + "debugid", + "failure", + "serde", + "serde_json", + "url", + "uuid", +] + +[[package]] +name = "serde" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_qs" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5cb0f0564a84554436c4ceff5c896308d4e09d0eb4bd0215b8f698f88084601" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + +[[package]] +name = "serde_url_params" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +dependencies = [ + "serde", + "url", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha-1" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" + +[[package]] +name = "sized-chunks" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59044ea371ad781ff976f7b06480b9f0180e834eda94114f2afb4afc12b7718" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "slice-group-by" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7474f0b646d228360ab62ed974744617bc869d959eac8403bfa3665931a7fb" + +[[package]] +name = "smallstr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e922794d168678729ffc7e07182721a14219c65814e66e91b839a272fe5ae4f" +dependencies = [ + "serde", + "smallvec", +] + +[[package]] +name = "smallvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" + +[[package]] +name = "snap" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98d3306e84bf86710d6cd8b4c9c3b721d5454cc91a603180f8f8cd06cfd317b4" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "standback" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" +dependencies = [ + "version_check", +] + +[[package]] +name = "stderrlog" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02f316286ae558d83acc93dd81eaba096e746987a7961d4a9ae026842bae67f" +dependencies = [ + "atty", + "chrono", + "log", + "termcolor", + "thread_local", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a571a711dddd09019ccc628e1b17fe87c59b09d513c06c026877aa708334f37a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synchronoise" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tar" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +dependencies = [ + "filetime", + "libc", + "redox_syscall", + "xattr", +] + +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "term_size" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "term_size", + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-uds", + "num_cpus", + "pin-project-lite 0.1.11", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" +dependencies = [ + "futures-core", + "rustls 0.18.1", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.1.11", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite 0.2.0", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "trust-dns-proto" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" +dependencies = [ + "async-trait", + "backtrace", + "enum-as-inner", + "futures", + "idna", + "lazy_static", + "log", + "rand 0.7.3", + "smallvec", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" +dependencies = [ + "backtrace", + "cfg-if 0.1.10", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "uname" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" +dependencies = [ + "libc", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "294b85ef5dbc3670a72e82a89971608a1fcc4ed5c7c5a2895230d31a95f0569b" +dependencies = [ + "base64 0.13.0", + "chunked_transfer", + "log", + "once_cell", + "qstring", + "rustls 0.19.0", + "url", + "webpki", + "webpki-roots 0.21.0", +] + +[[package]] +name = "url" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-width" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" + +[[package]] +name = "uuid" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" +dependencies = [ + "rand 0.7.3", + "serde", +] + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "vergen" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +dependencies = [ + "bitflags", + "chrono", +] + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi 0.3.9", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "web-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" +dependencies = [ + "webpki", +] + +[[package]] +name = "webpki-roots" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +dependencies = [ + "webpki", +] + +[[package]] +name = "whoami" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35495e7faf4c657051a8e9725d9c37ac57879e915be3ed55bb401af84382035" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "widestring" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + +[[package]] +name = "zerocopy" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + +[[package]] +name = "zstd" +version = "0.5.4+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "2.0.6+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.4.18+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" +dependencies = [ + "cc", + "glob", + "itertools", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index 6818443e6..150005d66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,20 +1,16 @@ [package] -name = "meilisearch-http" +authors = ["Quentin de Quelen ", "Clément Renault "] description = "MeiliSearch HTTP server" -version = "0.17.0" -license = "MIT" -authors = [ - "Quentin de Quelen ", - "Clément Renault ", -] edition = "2018" - +license = "MIT" +name = "meilisearch-http" +version = "0.17.0" [[bin]] name = "meilisearch" path = "src/main.rs" -[features] -default = ["sentry"] +[build-dependencies] +vergen = "3.1.0" [dependencies] actix-cors = "0.5.3" @@ -22,6 +18,7 @@ actix-http = "2" actix-rt = "1" actix-service = "1.0.6" actix-web = { version = "3.3.2", features = ["rustls"] } +byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } bytes = "0.6.0" chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.0" @@ -29,19 +26,16 @@ env_logger = "0.8.2" flate2 = "1.0.18" futures = "0.3.7" http = "0.2.1" -indexmap = { version = "1.3.2", features = ["serde-1"] } +indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" main_error = "0.1.0" -meilisearch-core = { path = "../meilisearch-core", version = "0.17.0" } -meilisearch-error = { path = "../meilisearch-error", version = "0.17.0" } -meilisearch-schema = { path = "../meilisearch-schema", version = "0.17.0" } -meilisearch-tokenizer = {path = "../meilisearch-tokenizer", version = "0.17.0"} +milli = { path = "../milli" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" regex = "1.4.2" rustls = "0.18" -serde = { version = "1.0.117", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.59", features = ["preserve_order"] } serde_qs = "0.8.1" sha2 = "0.9.1" @@ -50,37 +44,27 @@ slice-group-by = "0.2.6" structopt = "0.3.20" tar = "0.4.29" tempfile = "3.1.0" -tokio = { version = "0.2.18", features = ["macros"] } -ureq = { version = "1.5.1", features = ["tls"], default-features = false } +tokio = "*" +ureq = { version = "1.5.1", default-features = false, features = ["tls"] } walkdir = "2.3.1" whoami = "1.0.0" +meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } [dependencies.sentry] -version = "0.18.1" default-features = false -features = [ - "with_client_implementation", - "with_panic", - "with_failure", - "with_device_info", - "with_rust_info", - "with_reqwest_transport", - "with_rustls", - "with_env_logger" -] +features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] optional = true +version = "0.18.1" + [dev-dependencies] serde_url_params = "0.2.0" tempdir = "0.3.7" +assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } tokio = { version = "0.2.18", features = ["macros", "time"] } -[dev-dependencies.assert-json-diff] -git = "https://github.com/qdequele/assert-json-diff" -branch = "master" - -[build-dependencies] -vergen = "3.1.0" +[features] +default = ["sentry"] [target.'cfg(unix)'.dependencies] jemallocator = "0.3.2" diff --git a/src/data.rs b/src/data.rs index 783c81fd8..2b0ece53a 100644 --- a/src/data.rs +++ b/src/data.rs @@ -3,12 +3,11 @@ use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; -use meilisearch_core::{Database, DatabaseOptions, Index}; use sha2::Digest; +use milli::Index; -use crate::error::{Error as MSError, ResponseError}; -use crate::index_update_callback; use crate::option::Opt; +use crate::updates::UpdateQueue; #[derive(Clone)] pub struct Data { @@ -25,7 +24,8 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { - pub db: Arc, + pub indexes: Arc, + pub update_store: UpdateQueue, pub db_path: String, pub dumps_dir: PathBuf, pub dump_batch_size: usize, @@ -59,104 +59,7 @@ impl ApiKeys { } impl Data { - pub fn new(opt: Opt) -> Result> { - let db_path = opt.db_path.clone(); - let dumps_dir = opt.dumps_dir.clone(); - let dump_batch_size = opt.dump_batch_size; - let server_pid = std::process::id(); - - let db_opt = DatabaseOptions { - main_map_size: opt.max_mdb_size, - update_map_size: opt.max_udb_size, - }; - - let http_payload_size_limit = opt.http_payload_size_limit; - - let db = Arc::new(Database::open_or_create(opt.db_path, db_opt)?); - - let mut api_keys = ApiKeys { - master: opt.master_key, - private: None, - public: None, - }; - - api_keys.generate_missing_api_keys(); - - let inner_data = DataInner { - db: db.clone(), - db_path, - dumps_dir, - dump_batch_size, - api_keys, - server_pid, - http_payload_size_limit, - }; - - let data = Data { - inner: Arc::new(inner_data), - }; - - let callback_context = data.clone(); - db.set_update_callback(Box::new(move |index_uid, status| { - index_update_callback(&index_uid, &callback_context, status); - })); - - Ok(data) - } - - fn create_index(&self, uid: &str) -> Result { - if !uid - .chars() - .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') - { - return Err(MSError::InvalidIndexUid.into()); - } - - let created_index = self.db.create_index(&uid).map_err(|e| match e { - meilisearch_core::Error::IndexAlreadyExists => e.into(), - _ => ResponseError::from(MSError::create_index(e)), - })?; - - self.db.main_write::<_, _, ResponseError>(|mut writer| { - created_index.main.put_name(&mut writer, uid)?; - - created_index - .main - .created_at(&writer)? - .ok_or(MSError::internal("Impossible to read created at"))?; - - created_index - .main - .updated_at(&writer)? - .ok_or(MSError::internal("Impossible to read updated at"))?; - Ok(()) - })?; - - Ok(created_index) - } - - pub fn get_or_create_index(&self, uid: &str, f: F) -> Result - where - F: FnOnce(&Index) -> Result, - { - let mut index_has_been_created = false; - - let index = match self.db.open_index(&uid) { - Some(index) => index, - None => { - index_has_been_created = true; - self.create_index(&uid)? - } - }; - - match f(&index) { - Ok(r) => Ok(r), - Err(err) => { - if index_has_been_created { - let _ = self.db.delete_index(&uid); - } - Err(err) - } - } + pub fn new(_opt: Opt) -> Result> { + todo!() } } diff --git a/src/dump.rs b/src/dump.rs index 468dbf640..544fffaa7 100644 --- a/src/dump.rs +++ b/src/dump.rs @@ -8,9 +8,6 @@ use actix_web::web; use chrono::offset::Utc; use indexmap::IndexMap; use log::{error, info}; -use meilisearch_core::{MainWriter, MainReader, UpdateReader}; -use meilisearch_core::settings::Settings; -use meilisearch_core::update::{apply_settings_update, apply_documents_addition}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -20,6 +17,7 @@ use crate::Data; use crate::error::{Error, ResponseError}; use crate::helpers::compression; use crate::routes::index; +use crate::routes::setting::Settings; use crate::routes::index::IndexResponse; // Mutex to share dump progress. diff --git a/src/error.rs b/src/error.rs index e779c5708..c3533bcef 100644 --- a/src/error.rs +++ b/src/error.rs @@ -6,9 +6,9 @@ use actix_web as aweb; use actix_web::error::{JsonPayloadError, QueryPayloadError}; use actix_web::http::StatusCode; use serde::ser::{Serialize, Serializer, SerializeStruct}; - use meilisearch_error::{ErrorCode, Code}; + #[derive(Debug)] pub struct ResponseError { inner: Box, @@ -34,18 +34,6 @@ impl From for ResponseError { } } -impl From for ResponseError { - fn from(err: meilisearch_core::Error) -> ResponseError { - ResponseError { inner: Box::new(err) } - } -} - -impl From for ResponseError { - fn from(err: meilisearch_schema::Error) -> ResponseError { - ResponseError { inner: Box::new(err) } - } -} - impl From for ResponseError { fn from(err: FacetCountError) -> ResponseError { ResponseError { inner: Box::new(err) } @@ -123,8 +111,9 @@ impl ErrorCode for Error { SearchDocuments(_) => Code::SearchDocuments, PayloadTooLarge => Code::PayloadTooLarge, UnsupportedMediaType => Code::UnsupportedMediaType, - DumpAlreadyInProgress => Code::DumpAlreadyInProgress, - DumpProcessFailed(_) => Code::DumpProcessFailed, + _ => unreachable!() + //DumpAlreadyInProgress => Code::DumpAlreadyInProgress, + //DumpProcessFailed(_) => Code::DumpProcessFailed, } } } @@ -270,12 +259,6 @@ impl From for Error { } } -impl From for Error { - fn from(err: meilisearch_core::Error) -> Error { - Error::Internal(err.to_string()) - } -} - impl From for Error { fn from(err: serde_json::error::Error) -> Error { Error::Internal(err.to_string()) diff --git a/src/lib.rs b/src/lib.rs index b5f35f277..0f2aabccd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,24 +3,20 @@ pub mod data; pub mod error; pub mod helpers; -pub mod models; pub mod option; pub mod routes; -pub mod analytics; -pub mod snapshot; -pub mod dump; +mod updates; +//pub mod analytics; +//pub mod snapshot; +//pub mod dump; use actix_http::Error; use actix_service::ServiceFactory; use actix_web::{dev, web, App}; -use chrono::Utc; -use log::error; - -use meilisearch_core::{Index, MainWriter, ProcessedUpdateResult}; pub use option::Opt; pub use self::data::Data; -use self::error::{payload_error_handler, ResponseError}; +use self::error::payload_error_handler; pub fn create_app( data: &Data, @@ -55,8 +51,8 @@ pub fn create_app( .configure(routes::synonym::services) .configure(routes::health::services) .configure(routes::stats::services) - .configure(routes::key::services) - .configure(routes::dump::services); + .configure(routes::key::services); + //.configure(routes::dump::services); if enable_frontend { app .service(routes::load_html) @@ -65,40 +61,3 @@ pub fn create_app( app } } - -pub fn index_update_callback_txn(index: Index, index_uid: &str, data: &Data, mut writer: &mut MainWriter) -> Result<(), String> { - if let Err(e) = data.db.compute_stats(&mut writer, index_uid) { - return Err(format!("Impossible to compute stats; {}", e)); - } - - if let Err(e) = data.db.set_last_update(&mut writer, &Utc::now()) { - return Err(format!("Impossible to update last_update; {}", e)); - } - - if let Err(e) = index.main.put_updated_at(&mut writer) { - return Err(format!("Impossible to update updated_at; {}", e)); - } - - Ok(()) -} - -pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) { - if status.error.is_some() { - return; - } - - if let Some(index) = data.db.open_index(index_uid) { - let db = &data.db; - let res = db.main_write::<_, _, ResponseError>(|mut writer| { - if let Err(e) = index_update_callback_txn(index, index_uid, data, &mut writer) { - error!("{}", e); - } - - Ok(()) - }); - match res { - Ok(_) => (), - Err(e) => error!("{}", e), - } - } -} diff --git a/src/main.rs b/src/main.rs index 800e2760c..e5b167cd0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,13 @@ -use std::{env, thread}; +use std::env; use actix_cors::Cors; use actix_web::{middleware, HttpServer}; use main_error::MainError; use meilisearch_http::helpers::NormalizePath; -use meilisearch_http::{create_app, index_update_callback, Data, Opt}; +use meilisearch_http::{create_app, Data, Opt}; use structopt::StructOpt; -use meilisearch_http::{snapshot, dump}; -mod analytics; +//mod analytics; #[cfg(target_os = "linux")] #[global_allocator] @@ -52,31 +51,25 @@ async fn main() -> Result<(), MainError> { _ => unreachable!(), } - if let Some(path) = &opt.import_snapshot { - snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; - } + //if let Some(path) = &opt.import_snapshot { + //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; + //} let data = Data::new(opt.clone())?; - if !opt.no_analytics { - let analytics_data = data.clone(); - let analytics_opt = opt.clone(); - thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); - } + //if !opt.no_analytics { + //let analytics_data = data.clone(); + //let analytics_opt = opt.clone(); + //thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); + //} - let data_cloned = data.clone(); - data.db.set_update_callback(Box::new(move |name, status| { - index_update_callback(name, &data_cloned, status); - })); + //if let Some(path) = &opt.import_dump { + //dump::import_dump(&data, path, opt.dump_batch_size)?; + //} - - if let Some(path) = &opt.import_dump { - dump::import_dump(&data, path, opt.dump_batch_size)?; - } - - if opt.schedule_snapshot { - snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; - } + //if opt.schedule_snapshot { + //snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; + //} print_launch_resume(&opt, &data); diff --git a/src/option.rs b/src/option.rs index e1d74fd63..cf505e1fe 100644 --- a/src/option.rs +++ b/src/option.rs @@ -3,6 +3,7 @@ use std::io::{BufReader, Read}; use std::path::PathBuf; use std::sync::Arc; +use byte_unit::Byte; use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; use rustls::{ AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, @@ -12,7 +13,7 @@ use structopt::StructOpt; const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; -#[derive(Debug, Default, Clone, StructOpt)] +#[derive(Debug, Clone, StructOpt)] pub struct Opt { /// The destination where the database must be created. #[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")] @@ -49,16 +50,16 @@ pub struct Opt { pub no_analytics: bool, /// The maximum size, in bytes, of the main lmdb database directory - #[structopt(long, env = "MEILI_MAX_MDB_SIZE", default_value = "107374182400")] // 100GB - pub max_mdb_size: usize, + #[structopt(long, env = "MEILI_MAX_MDB_SIZE", default_value = "100 GiB")] + pub max_mdb_size: Byte, /// The maximum size, in bytes, of the update lmdb database directory - #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "107374182400")] // 100GB - pub max_udb_size: usize, + #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "10 GiB")] + pub max_udb_size: Byte, /// The maximum size, in bytes, of accepted JSON payloads - #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "10485760")] // 10MB - pub http_payload_size_limit: usize, + #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "10 MiB")] + pub http_payload_size_limit: Byte, /// Read server certificates from CERTFILE. /// This should contain PEM-format certificates diff --git a/src/routes/document.rs b/src/routes/document.rs index 3c4b9acf5..3dfa36880 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -1,23 +1,20 @@ -use std::collections::{BTreeSet, HashSet}; - use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use indexmap::IndexMap; -use meilisearch_core::{update, MainReader}; use serde_json::Value; use serde::Deserialize; use crate::Data; -use crate::error::{Error, ResponseError}; +use crate::error::ResponseError; use crate::helpers::Authentication; -use crate::routes::{IndexParam, IndexUpdateResponse}; +use crate::routes::IndexParam; type Document = IndexMap; #[derive(Deserialize)] struct DocumentParam { - index_uid: String, - document_id: String, + _index_uid: String, + _document_id: String, } pub fn services(cfg: &mut web::ServiceConfig) { @@ -35,8 +32,8 @@ pub fn services(cfg: &mut web::ServiceConfig) { wrap = "Authentication::Public" )] async fn get_document( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -46,8 +43,8 @@ async fn get_document( wrap = "Authentication::Private" )] async fn delete_document( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -55,62 +52,51 @@ async fn delete_document( #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct BrowseQuery { - offset: Option, - limit: Option, - attributes_to_retrieve: Option, -} - -pub fn get_all_documents_sync( - data: &web::Data, - reader: &MainReader, - index_uid: &str, - offset: usize, - limit: usize, - attributes_to_retrieve: Option<&String> -) -> Result, Error> { - todo!() + _offset: Option, + _limit: Option, + _attributes_to_retrieve: Option, } #[get("/indexes/{index_uid}/documents", wrap = "Authentication::Public")] async fn get_all_documents( - data: web::Data, - path: web::Path, - params: web::Query, + _data: web::Data, + _path: web::Path, + _params: web::Query, ) -> Result { todo!() } -fn find_primary_key(document: &IndexMap) -> Option { - for key in document.keys() { - if key.to_lowercase().contains("id") { - return Some(key.to_string()); - } - } - None -} +//fn find_primary_key(document: &IndexMap) -> Option { + //for key in document.keys() { + //if key.to_lowercase().contains("id") { + //return Some(key.to_string()); + //} + //} + //None +//} #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateDocumentsQuery { - primary_key: Option, + _primary_key: Option, } async fn update_multiple_documents( - data: web::Data, - path: web::Path, - params: web::Query, - body: web::Json>, - is_partial: bool, + _data: web::Data, + _path: web::Path, + _params: web::Query, + _body: web::Json>, + _is_partial: bool, ) -> Result { todo!() } #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn add_documents( - data: web::Data, - path: web::Path, - params: web::Query, - body: web::Json>, + _data: web::Data, + _path: web::Path, + _params: web::Query, + _body: web::Json>, ) -> Result { todo!() } @@ -130,17 +116,17 @@ async fn update_documents( wrap = "Authentication::Private" )] async fn delete_documents( - data: web::Data, - path: web::Path, - body: web::Json>, + _data: web::Data, + _path: web::Path, + _body: web::Json>, ) -> Result { todo!() } #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn clear_all_documents( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } diff --git a/src/routes/index.rs b/src/routes/index.rs index 9a00030d0..36860f34f 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -1,14 +1,10 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; -use log::error; -use meilisearch_core::{Database, MainReader, UpdateReader}; -use meilisearch_core::update::UpdateStatus; -use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use crate::Data; -use crate::error::{Error, ResponseError}; +use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; @@ -22,10 +18,6 @@ pub fn services(cfg: &mut web::ServiceConfig) { .service(get_all_updates_status); } -fn generate_uid() -> String { - todo!() -} - #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexResponse { @@ -36,22 +28,15 @@ pub struct IndexResponse { pub primary_key: Option, } -pub fn list_indexes_sync(data: &web::Data, reader: &MainReader) -> Result, ResponseError> { - todo!() -} - #[get("/indexes", wrap = "Authentication::Private")] -async fn list_indexes(data: web::Data) -> Result { - let reader = data.db.main_read_txn()?; - let indexes = list_indexes_sync(&data, &reader)?; - - Ok(HttpResponse::Ok().json(indexes)) +async fn list_indexes(_data: web::Data) -> Result { + todo!() } #[get("/indexes/{index_uid}", wrap = "Authentication::Private")] async fn get_index( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -64,20 +49,10 @@ struct IndexCreateRequest { primary_key: Option, } - -pub fn create_index_sync( - database: &std::sync::Arc, - uid: String, - name: String, - primary_key: Option, -) -> Result { - todo!() -} - #[post("/indexes", wrap = "Authentication::Private")] async fn create_index( - data: web::Data, - body: web::Json, + _data: web::Data, + _body: web::Json, ) -> Result { todo!() } @@ -101,25 +76,25 @@ struct UpdateIndexResponse { #[put("/indexes/{index_uid}", wrap = "Authentication::Private")] async fn update_index( - data: web::Data, - path: web::Path, - body: web::Json, + _data: web::Data, + _path: web::Path, + _body: web::Json, ) -> Result { todo!() } #[delete("/indexes/{index_uid}", wrap = "Authentication::Private")] async fn delete_index( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } #[derive(Deserialize)] struct UpdateParam { - index_uid: String, - update_id: u64, + _index_uid: String, + _update_id: u64, } #[get( @@ -127,23 +102,16 @@ struct UpdateParam { wrap = "Authentication::Private" )] async fn get_update_status( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } -pub fn get_all_updates_status_sync( - data: &web::Data, - reader: &UpdateReader, - index_uid: &str, -) -> Result, Error> { - todo!() -} #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] async fn get_all_updates_status( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } diff --git a/src/routes/key.rs b/src/routes/key.rs index f4f313e4e..be089beb2 100644 --- a/src/routes/key.rs +++ b/src/routes/key.rs @@ -17,6 +17,6 @@ struct KeysResponse { } #[get("/keys", wrap = "Authentication::Admin")] -async fn list(data: web::Data) -> HttpResponse { +async fn list(_data: web::Data) -> HttpResponse { todo!() } diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 15a858055..71cb2a8d8 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -10,11 +10,11 @@ pub mod setting; pub mod stats; pub mod stop_words; pub mod synonym; -pub mod dump; +//pub mod dump; #[derive(Deserialize)] pub struct IndexParam { - index_uid: String, + _index_uid: String, } #[derive(Serialize)] diff --git a/src/routes/search.rs b/src/routes/search.rs index 8d8f750a9..2c510e0d4 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -1,19 +1,12 @@ -use std::collections::{HashMap, HashSet}; - use actix_web::{get, post, web, HttpResponse}; -use log::warn; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::error::{Error, FacetCountError, ResponseError}; -use crate::helpers::meilisearch::{IndexSearchExt, SearchResult}; +use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; use crate::Data; -use meilisearch_core::facets::FacetFilter; -use meilisearch_schema::{FieldId, Schema}; - pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(search_with_post).service(search_with_url_query); } @@ -36,9 +29,9 @@ pub struct SearchQuery { #[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")] async fn search_with_url_query( - data: web::Data, - path: web::Path, - params: web::Query, + _data: web::Data, + _path: web::Path, + _params: web::Query, ) -> Result { todo!() } @@ -46,53 +39,24 @@ async fn search_with_url_query( #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchQueryPost { - q: Option, - offset: Option, - limit: Option, - attributes_to_retrieve: Option>, - attributes_to_crop: Option>, - crop_length: Option, - attributes_to_highlight: Option>, - filters: Option, - matches: Option, - facet_filters: Option, - facets_distribution: Option>, -} - -impl From for SearchQuery { - fn from(other: SearchQueryPost) -> SearchQuery { - SearchQuery { - q: other.q, - offset: other.offset, - limit: other.limit, - attributes_to_retrieve: other.attributes_to_retrieve.map(|attrs| attrs.join(",")), - attributes_to_crop: other.attributes_to_crop.map(|attrs| attrs.join(",")), - crop_length: other.crop_length, - attributes_to_highlight: other.attributes_to_highlight.map(|attrs| attrs.join(",")), - filters: other.filters, - matches: other.matches, - facet_filters: other.facet_filters.map(|f| f.to_string()), - facets_distribution: other.facets_distribution.map(|f| format!("{:?}", f)), - } - } + _q: Option, + _offset: Option, + _limit: Option, + _attributes_to_retrieve: Option>, + _attributes_to_crop: Option>, + _crop_length: Option, + _attributes_to_highlight: Option>, + _filters: Option, + _matches: Option, + _facet_filters: Option, + _facets_distribution: Option>, } #[post("/indexes/{index_uid}/search", wrap = "Authentication::Public")] async fn search_with_post( - data: web::Data, - path: web::Path, - params: web::Json, + _data: web::Data, + _path: web::Path, + _params: web::Json, ) -> Result { - let query: SearchQuery = params.0.into(); - let search_result = query.search(&path.index_uid, data)?; - Ok(HttpResponse::Ok().json(search_result)) -} - -impl SearchQuery { - fn search( - &self, - index_uid: &str, - data: web::Data, - ) -> Result { todo!() } diff --git a/src/routes/setting.rs b/src/routes/setting.rs index 36c7fd239..a405de45a 100644 --- a/src/routes/setting.rs +++ b/src/routes/setting.rs @@ -1,15 +1,12 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use actix_web::{delete, get, post}; use actix_web::{web, HttpResponse}; -use meilisearch_core::{MainReader, UpdateWriter}; -use meilisearch_core::settings::{Settings, SettingsUpdate, UpdateState, DEFAULT_RANKING_RULES}; -use meilisearch_schema::Schema; use crate::Data; -use crate::error::{Error, ResponseError}; +use crate::error::ResponseError; use crate::helpers::Authentication; -use crate::routes::{IndexParam, IndexUpdateResponse}; +use crate::updates::Settings; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(update_all) @@ -32,27 +29,28 @@ pub fn services(cfg: &mut web::ServiceConfig) { .service(update_attributes_for_faceting); } + #[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] async fn update_all( - data: web::Data, - path: web::Path, - body: web::Json, + _data: web::Data, + _path: web::Path, + _body: web::Json, ) -> Result { todo!() } #[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] async fn get_all( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } #[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] async fn delete_all( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -62,8 +60,8 @@ async fn delete_all( wrap = "Authentication::Private" )] async fn get_rules( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -73,9 +71,9 @@ async fn get_rules( wrap = "Authentication::Private" )] async fn update_rules( - data: web::Data, - path: web::Path, - body: web::Json>>, + _data: web::Data, + _path: web::Path, + _body: web::Json>>, ) -> Result { todo!() } @@ -85,8 +83,8 @@ async fn update_rules( wrap = "Authentication::Private" )] async fn delete_rules( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -96,8 +94,8 @@ async fn delete_rules( wrap = "Authentication::Private" )] async fn get_distinct( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -107,9 +105,9 @@ async fn get_distinct( wrap = "Authentication::Private" )] async fn update_distinct( - data: web::Data, - path: web::Path, - body: web::Json>, + _data: web::Data, + _path: web::Path, + _body: web::Json>, ) -> Result { todo!() } @@ -119,8 +117,8 @@ async fn update_distinct( wrap = "Authentication::Private" )] async fn delete_distinct( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -130,8 +128,8 @@ async fn delete_distinct( wrap = "Authentication::Private" )] async fn get_searchable( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -141,9 +139,9 @@ async fn get_searchable( wrap = "Authentication::Private" )] async fn update_searchable( - data: web::Data, - path: web::Path, - body: web::Json>>, + _data: web::Data, + _path: web::Path, + _body: web::Json>>, ) -> Result { todo!() } @@ -153,8 +151,8 @@ async fn update_searchable( wrap = "Authentication::Private" )] async fn delete_searchable( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -164,8 +162,8 @@ async fn delete_searchable( wrap = "Authentication::Private" )] async fn get_displayed( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -175,9 +173,9 @@ async fn get_displayed( wrap = "Authentication::Private" )] async fn update_displayed( - data: web::Data, - path: web::Path, - body: web::Json>>, + _data: web::Data, + _path: web::Path, + _body: web::Json>>, ) -> Result { todo!() } @@ -187,8 +185,8 @@ async fn update_displayed( wrap = "Authentication::Private" )] async fn delete_displayed( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -198,8 +196,8 @@ async fn delete_displayed( wrap = "Authentication::Private" )] async fn get_attributes_for_faceting( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -209,9 +207,9 @@ async fn get_attributes_for_faceting( wrap = "Authentication::Private" )] async fn update_attributes_for_faceting( - data: web::Data, - path: web::Path, - body: web::Json>>, + _data: web::Data, + _path: web::Path, + _body: web::Json>>, ) -> Result { todo!() } @@ -221,8 +219,8 @@ async fn update_attributes_for_faceting( wrap = "Authentication::Private" )] async fn delete_attributes_for_faceting( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } diff --git a/src/routes/stats.rs b/src/routes/stats.rs index 734811450..cba64194b 100644 --- a/src/routes/stats.rs +++ b/src/routes/stats.rs @@ -4,11 +4,9 @@ use actix_web::web; use actix_web::HttpResponse; use actix_web::get; use chrono::{DateTime, Utc}; -use log::error; use serde::Serialize; -use walkdir::WalkDir; -use crate::error::{Error, ResponseError}; +use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; use crate::Data; @@ -29,8 +27,8 @@ struct IndexStatsResponse { #[get("/indexes/{index_uid}/stats", wrap = "Authentication::Private")] async fn index_stats( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -44,7 +42,7 @@ struct StatsResult { } #[get("/stats", wrap = "Authentication::Private")] -async fn get_stats(data: web::Data) -> Result { +async fn get_stats(_data: web::Data) -> Result { todo!() } diff --git a/src/routes/stop_words.rs b/src/routes/stop_words.rs index f57d9cdd7..5b116e829 100644 --- a/src/routes/stop_words.rs +++ b/src/routes/stop_words.rs @@ -1,11 +1,10 @@ use actix_web::{web, HttpResponse}; use actix_web::{delete, get, post}; -use meilisearch_core::settings::{SettingsUpdate, UpdateState}; use std::collections::BTreeSet; -use crate::error::{Error, ResponseError}; +use crate::error::ResponseError; use crate::helpers::Authentication; -use crate::routes::{IndexParam, IndexUpdateResponse}; +use crate::routes::IndexParam; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { @@ -17,8 +16,8 @@ pub fn services(cfg: &mut web::ServiceConfig) { wrap = "Authentication::Private" )] async fn get( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -28,9 +27,9 @@ async fn get( wrap = "Authentication::Private" )] async fn update( - data: web::Data, - path: web::Path, - body: web::Json>, + _data: web::Data, + _path: web::Path, + _body: web::Json>, ) -> Result { todo!() } @@ -40,8 +39,8 @@ async fn update( wrap = "Authentication::Private" )] async fn delete( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } diff --git a/src/routes/synonym.rs b/src/routes/synonym.rs index 398889473..43a0e4aeb 100644 --- a/src/routes/synonym.rs +++ b/src/routes/synonym.rs @@ -2,12 +2,10 @@ use std::collections::BTreeMap; use actix_web::{web, HttpResponse}; use actix_web::{delete, get, post}; -use indexmap::IndexMap; -use meilisearch_core::settings::{SettingsUpdate, UpdateState}; -use crate::error::{Error, ResponseError}; +use crate::error::ResponseError; use crate::helpers::Authentication; -use crate::routes::{IndexParam, IndexUpdateResponse}; +use crate::routes::IndexParam; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { @@ -19,8 +17,8 @@ pub fn services(cfg: &mut web::ServiceConfig) { wrap = "Authentication::Private" )] async fn get( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } @@ -30,9 +28,9 @@ async fn get( wrap = "Authentication::Private" )] async fn update( - data: web::Data, - path: web::Path, - body: web::Json>>, + _data: web::Data, + _path: web::Path, + _body: web::Json>>, ) -> Result { todo!() } @@ -42,8 +40,8 @@ async fn update( wrap = "Authentication::Private" )] async fn delete( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } diff --git a/src/updates/mod.rs b/src/updates/mod.rs new file mode 100644 index 000000000..249bec6d2 --- /dev/null +++ b/src/updates/mod.rs @@ -0,0 +1,17 @@ +mod settings; + +pub use settings::{Settings, Facets}; + +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +enum UpdateMeta { + DocumentsAddition { method: String, format: String }, + ClearDocuments, + Settings(Settings), + Facets(Facets), +} + +#[derive(Clone, Debug)] +pub struct UpdateQueue; diff --git a/src/updates/settings.rs b/src/updates/settings.rs new file mode 100644 index 000000000..55c0cccf2 --- /dev/null +++ b/src/updates/settings.rs @@ -0,0 +1,51 @@ +use std::num::NonZeroUsize; +use std::collections::HashMap; + +use serde::{Serialize, Deserialize, de::Deserializer}; + +// Any value that is present is considered Some value, including null. +fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> +where T: Deserialize<'de>, + D: Deserializer<'de> +{ + Deserialize::deserialize(deserializer).map(Some) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Settings { + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + displayed_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + searchable_attributes: Option>>, + + #[serde(default)] + faceted_attributes: Option>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + criteria: Option>>, +} + + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Facets { + level_group_size: Option, + min_level_size: Option, +} + From 55e15529577d2b04244d554188112fab3d85d1d2 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 22 Dec 2020 17:13:50 +0100 Subject: [PATCH 004/522] update queue refactor, first iteration --- Cargo.lock | 5 + Cargo.toml | 8 +- src/option.rs | 7 +- src/updates/mod.rs | 336 +++++++++++++++++++++++++++++++++++++++- src/updates/settings.rs | 12 +- 5 files changed, 357 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b7386594..610c83318 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1142,6 +1142,7 @@ dependencies = [ "lmdb-rkv-sys", "once_cell", "page_size", + "serde", "synchronoise", "url", "zerocopy", @@ -1527,6 +1528,7 @@ dependencies = [ "actix-rt", "actix-service", "actix-web", + "anyhow", "assert-json-diff", "byte-unit", "bytes 0.6.0", @@ -1535,6 +1537,8 @@ dependencies = [ "env_logger 0.8.2", "flate2", "futures", + "grenad", + "heed", "http", "indexmap", "jemallocator", @@ -1545,6 +1549,7 @@ dependencies = [ "mime", "once_cell", "rand 0.7.3", + "rayon", "regex", "rustls 0.18.1", "sentry", diff --git a/Cargo.toml b/Cargo.toml index 150005d66..cc25f0c8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,21 +18,26 @@ actix-http = "2" actix-rt = "1" actix-service = "1.0.6" actix-web = { version = "3.3.2", features = ["rustls"] } +anyhow = "1.0.36" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } bytes = "0.6.0" chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.0" env_logger = "0.8.2" -flate2 = "1.0.18" +flate2 = "1.0.19" futures = "0.3.7" +grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } +heed = "0.10.6" http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" main_error = "0.1.0" +meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } milli = { path = "../milli" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" +rayon = "1.5.0" regex = "1.4.2" rustls = "0.18" serde = { version = "1.0", features = ["derive"] } @@ -48,7 +53,6 @@ tokio = "*" ureq = { version = "1.5.1", default-features = false, features = ["tls"] } walkdir = "2.3.1" whoami = "1.0.0" -meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } [dependencies.sentry] default-features = false diff --git a/src/option.rs b/src/option.rs index cf505e1fe..f9e98f4fa 100644 --- a/src/option.rs +++ b/src/option.rs @@ -11,13 +11,15 @@ use rustls::{ }; use structopt::StructOpt; +use crate::updates::IndexerOpts; + const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; #[derive(Debug, Clone, StructOpt)] pub struct Opt { /// The destination where the database must be created. #[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")] - pub db_path: String, + pub db_path: PathBuf, /// The address on which the http server will listen. #[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")] @@ -132,6 +134,9 @@ pub struct Opt { /// The batch size used in the importation process, the bigger it is the faster the dump is created. #[structopt(long, env = "MEILI_DUMP_BATCH_SIZE", default_value = "1024")] pub dump_batch_size: usize, + + #[structopt(flatten)] + pub indexer_options: IndexerOpts, } impl Opt { diff --git a/src/updates/mod.rs b/src/updates/mod.rs index 249bec6d2..dd76ed1aa 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -2,7 +2,22 @@ mod settings; pub use settings::{Settings, Facets}; +use std::io; +use std::path::Path; +use std::sync::Arc; + +use anyhow::Result; +use flate2::read::GzDecoder; +use grenad::CompressionType; +use byte_unit::Byte; +use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod, UpdateIndexingStep::*}; +use milli::{UpdateStore, UpdateHandler as Handler, Index}; +use rayon::ThreadPool; use serde::{Serialize, Deserialize}; +use tokio::sync::broadcast; +use structopt::StructOpt; + +use crate::option::Opt; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] @@ -13,5 +28,322 @@ enum UpdateMeta { Facets(Facets), } -#[derive(Clone, Debug)] -pub struct UpdateQueue; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +enum UpdateMetaProgress { + DocumentsAddition { + step: usize, + total_steps: usize, + current: usize, + total: Option, + }, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "type")] +enum UpdateStatus { + Pending { update_id: u64, meta: M }, + Progressing { update_id: u64, meta: P }, + Processed { update_id: u64, meta: N }, + Aborted { update_id: u64, meta: M }, +} + +#[derive(Clone)] +pub struct UpdateQueue { + inner: Arc>, +} + + +#[derive(Debug, Clone, StructOpt)] +pub struct IndexerOpts { + /// The amount of documents to skip before printing + /// a log regarding the indexing advancement. + #[structopt(long, default_value = "100000")] // 100k + pub log_every_n: usize, + + /// MTBL max number of chunks in bytes. + #[structopt(long)] + pub max_nb_chunks: Option, + + /// The maximum amount of memory to use for the MTBL buffer. It is recommended + /// to use something like 80%-90% of the available memory. + /// + /// It is automatically split by the number of jobs e.g. if you use 7 jobs + /// and 7 GB of max memory, each thread will use a maximum of 1 GB. + #[structopt(long, default_value = "7 GiB")] + pub max_memory: Byte, + + /// Size of the linked hash map cache when indexing. + /// The bigger it is, the faster the indexing is but the more memory it takes. + #[structopt(long, default_value = "500")] + pub linked_hash_map_size: usize, + + /// The name of the compression algorithm to use when compressing intermediate + /// chunks during indexing documents. + /// + /// Choosing a fast algorithm will make the indexing faster but may consume more memory. + #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] + pub chunk_compression_type: CompressionType, + + /// The level of compression of the chosen algorithm. + #[structopt(long, requires = "chunk-compression-type")] + pub chunk_compression_level: Option, + + /// The number of bytes to remove from the begining of the chunks while reading/sorting + /// or merging them. + /// + /// File fusing must only be enable on file systems that support the `FALLOC_FL_COLLAPSE_RANGE`, + /// (i.e. ext4 and XFS). File fusing will only work if the `enable-chunk-fusing` is set. + #[structopt(long, default_value = "4 GiB")] + pub chunk_fusing_shrink_size: Byte, + + /// Enable the chunk fusing or not, this reduces the amount of disk used by a factor of 2. + #[structopt(long)] + pub enable_chunk_fusing: bool, + + /// Number of parallel jobs for indexing, defaults to # of CPUs. + #[structopt(long)] + pub indexing_jobs: Option, +} + +type UpdateSender = broadcast::Sender>; + +struct UpdateHandler { + indexes: Arc, + max_nb_chunks: Option, + chunk_compression_level: Option, + thread_pool: ThreadPool, + log_frequency: usize, + max_memory: usize, + linked_hash_map_size: usize, + chunk_compression_type: CompressionType, + chunk_fusing_shrink_size: u64, + update_status_sender: UpdateSender, +} + +impl UpdateHandler { + fn new( + opt: &IndexerOpts, + indexes: Arc, + update_status_sender: UpdateSender, + ) -> Result { + let thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(opt.indexing_jobs.unwrap_or(0)) + .build()?; + Ok(Self { + indexes, + max_nb_chunks: opt.max_nb_chunks, + chunk_compression_level: opt.chunk_compression_level, + thread_pool, + log_frequency: opt.log_every_n, + max_memory: opt.max_memory.get_bytes() as usize, + linked_hash_map_size: opt.linked_hash_map_size, + chunk_compression_type: opt.chunk_compression_type, + chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), + update_status_sender, + }) + } + + fn update_buidler(&self, update_id: u64) -> UpdateBuilder { + // We prepare the update by using the update builder. + let mut update_builder = UpdateBuilder::new(update_id); + if let Some(max_nb_chunks) = self.max_nb_chunks { + update_builder.max_nb_chunks(max_nb_chunks); + } + if let Some(chunk_compression_level) = self.chunk_compression_level { + update_builder.chunk_compression_level(chunk_compression_level); + } + update_builder.thread_pool(&self.thread_pool); + update_builder.log_every_n(self.log_frequency); + update_builder.max_memory(self.max_memory); + update_builder.linked_hash_map_size(self.linked_hash_map_size); + update_builder.chunk_compression_type(self.chunk_compression_type); + update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); + update_builder + } + + fn update_documents( + &self, + format: String, + method: String, + content: &[u8], + update_builder: UpdateBuilder, + ) -> Result<()> { + // We must use the write transaction of the update here. + let mut wtxn = self.indexes.write_txn()?; + let mut builder = update_builder.index_documents(&mut wtxn, &self.indexes); + + match format.as_str() { + "csv" => builder.update_format(UpdateFormat::Csv), + "json" => builder.update_format(UpdateFormat::Json), + "json-stream" => builder.update_format(UpdateFormat::JsonStream), + otherwise => panic!("invalid update format {:?}", otherwise), + }; + + match method.as_str() { + "replace" => builder.index_documents_method(IndexDocumentsMethod::ReplaceDocuments), + "update" => builder.index_documents_method(IndexDocumentsMethod::UpdateDocuments), + otherwise => panic!("invalid indexing method {:?}", otherwise), + }; + + let gzipped = true; + let reader = if gzipped { + Box::new(GzDecoder::new(content)) + } else { + Box::new(content) as Box + }; + + let result = builder.execute(reader, |indexing_step, update_id| { + let (current, total) = match indexing_step { + TransformFromUserIntoGenericFormat { documents_seen } => (documents_seen, None), + ComputeIdsAndMergeDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)), + IndexDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)), + MergeDataIntoFinalDatabase { databases_seen, total_databases } => (databases_seen, Some(total_databases)), + }; + let _ = self.update_status_sender.send(UpdateStatus::Progressing { + update_id, + meta: UpdateMetaProgress::DocumentsAddition { + step: indexing_step.step(), + total_steps: indexing_step.number_of_steps(), + current, + total, + } + }); + }); + + match result { + Ok(()) => wtxn.commit().map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn clear_documents(&self, update_builder: UpdateBuilder) -> Result<()> { + // We must use the write transaction of the update here. + let mut wtxn = self.indexes.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, &self.indexes); + + match builder.execute() { + Ok(_count) => wtxn.commit().map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn update_settings(&self, settings: Settings, update_builder: UpdateBuilder) -> Result<()> { + // We must use the write transaction of the update here. + let mut wtxn = self.indexes.write_txn()?; + let mut builder = update_builder.settings(&mut wtxn, &self.indexes); + + // We transpose the settings JSON struct into a real setting update. + if let Some(names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names), + None => builder.reset_searchable_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names), + None => builder.reset_displayed_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(facet_types) = settings.faceted_attributes { + builder.set_faceted_fields(facet_types); + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(criteria) = settings.criteria { + match criteria { + Some(criteria) => builder.set_criteria(criteria), + None => builder.reset_criteria(), + } + } + + let result = builder.execute(|indexing_step, update_id| { + let (current, total) = match indexing_step { + TransformFromUserIntoGenericFormat { documents_seen } => (documents_seen, None), + ComputeIdsAndMergeDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)), + IndexDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)), + MergeDataIntoFinalDatabase { databases_seen, total_databases } => (databases_seen, Some(total_databases)), + }; + let _ = self.update_status_sender.send(UpdateStatus::Progressing { + update_id, + meta: UpdateMetaProgress::DocumentsAddition { + step: indexing_step.step(), + total_steps: indexing_step.number_of_steps(), + current, + total, + } + }); + }); + + match result { + Ok(_count) => wtxn.commit().map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn update_facets(&self, levels: Facets, update_builder: UpdateBuilder) -> Result<()> { + // We must use the write transaction of the update here. + let mut wtxn = self.indexes.write_txn()?; + let mut builder = update_builder.facets(&mut wtxn, &self.indexes); + if let Some(value) = levels.level_group_size { + builder.level_group_size(value); + } + if let Some(value) = levels.min_level_size { + builder.min_level_size(value); + } + match builder.execute() { + Ok(()) => wtxn.commit().map_err(Into::into), + Err(e) => Err(e.into()) + } + } +} + +impl Handler for UpdateHandler { + fn handle_update(&mut self, update_id: u64, meta: UpdateMeta, content: &[u8]) -> heed::Result { + use UpdateMeta::*; + + let update_builder = self.update_buidler(update_id); + + let result: anyhow::Result<()> = match meta { + DocumentsAddition { method, format } => { + self.update_documents(format, method, content, update_builder) + }, + ClearDocuments => self.clear_documents(update_builder), + Settings(settings) => self.update_settings(settings, update_builder), + Facets(levels) => self.update_facets(levels, update_builder), + }; + + let meta = match result { + Ok(()) => format!("valid update content"), + Err(e) => format!("error while processing update content: {:?}", e), + }; + + let processed = UpdateStatus::Processed { update_id, meta: meta.clone() }; + let _ = self.update_status_sender.send(processed); + + Ok(meta) + } +} + +impl UpdateQueue { + pub fn new>( + opt: Opt, + indexes: Arc, + ) -> Result { + let (sender, _) = broadcast::channel(100); + let handler = UpdateHandler::new(&opt.indexer_options, indexes, sender)?; + let size = opt.max_udb_size.get_bytes() as usize; + let path = opt.db_path.join("updates.mdb"); + let inner = UpdateStore::open( + Some(size), + path, + handler + )?; + Ok(Self { inner }) + } +} diff --git a/src/updates/settings.rs b/src/updates/settings.rs index 55c0cccf2..58414152a 100644 --- a/src/updates/settings.rs +++ b/src/updates/settings.rs @@ -20,24 +20,24 @@ pub struct Settings { deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none", )] - displayed_attributes: Option>>, + pub displayed_attributes: Option>>, #[serde( default, deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none", )] - searchable_attributes: Option>>, + pub searchable_attributes: Option>>, #[serde(default)] - faceted_attributes: Option>, + pub faceted_attributes: Option>, #[serde( default, deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none", )] - criteria: Option>>, + pub criteria: Option>>, } @@ -45,7 +45,7 @@ pub struct Settings { #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct Facets { - level_group_size: Option, - min_level_size: Option, + pub level_group_size: Option, + pub min_level_size: Option, } From 0d7c4beecdd7e25e255c0d8ea651d4bdb2cfdbcf Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 22 Dec 2020 17:53:13 +0100 Subject: [PATCH 005/522] reimplement Data --- src/data.rs | 43 ++++++++++++++++++++++++++--------- src/helpers/authentication.rs | 14 ++++++------ src/lib.rs | 2 +- src/main.rs | 2 +- src/updates/mod.rs | 6 ++--- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/data.rs b/src/data.rs index 2b0ece53a..621d8cf2b 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,6 +1,4 @@ -use std::error::Error; use std::ops::Deref; -use std::path::PathBuf; use std::sync::Arc; use sha2::Digest; @@ -25,13 +23,9 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { pub indexes: Arc, - pub update_store: UpdateQueue, - pub db_path: String, - pub dumps_dir: PathBuf, - pub dump_batch_size: usize, - pub api_keys: ApiKeys, - pub server_pid: u32, - pub http_payload_size_limit: usize, + pub update_queue: UpdateQueue, + api_keys: ApiKeys, + options: Opt, } #[derive(Clone)] @@ -59,7 +53,34 @@ impl ApiKeys { } impl Data { - pub fn new(_opt: Opt) -> Result> { - todo!() + pub fn new(options: Opt) -> anyhow::Result { + let db_size = options.max_mdb_size.get_bytes() as usize; + let path = options.db_path.join("main"); + let indexes = Index::new(&path, Some(db_size))?; + let indexes = Arc::new(indexes); + + let update_queue = UpdateQueue::new(&options, indexes.clone())?; + + let mut api_keys = ApiKeys { + master: options.clone().master_key, + private: None, + public: None, + }; + + api_keys.generate_missing_api_keys(); + + let inner = DataInner { indexes, options, update_queue, api_keys }; + let inner = Arc::new(inner); + + Ok(Data { inner }) + } + + #[inline] + pub fn http_payload_size_limit(&self) -> usize { + self.options.http_payload_size_limit.get_bytes() as usize + } + + pub fn api_keys(&self) -> &ApiKeys { + &self.api_keys } } diff --git a/src/helpers/authentication.rs b/src/helpers/authentication.rs index 974c622f0..0de709ba3 100644 --- a/src/helpers/authentication.rs +++ b/src/helpers/authentication.rs @@ -65,7 +65,7 @@ where // it means that actix-web has an issue or someone changes the type `Data`. let data = req.app_data::>().unwrap(); - if data.api_keys.master.is_none() { + if data.api_keys().master.is_none() { return Box::pin(svc.call(req)); } @@ -80,15 +80,15 @@ where }; let authenticated = match self.acl { - Authentication::Admin => data.api_keys.master.as_deref() == Some(auth_header), + Authentication::Admin => data.api_keys().master.as_deref() == Some(auth_header), Authentication::Private => { - data.api_keys.master.as_deref() == Some(auth_header) - || data.api_keys.private.as_deref() == Some(auth_header) + data.api_keys().master.as_deref() == Some(auth_header) + || data.api_keys().private.as_deref() == Some(auth_header) } Authentication::Public => { - data.api_keys.master.as_deref() == Some(auth_header) - || data.api_keys.private.as_deref() == Some(auth_header) - || data.api_keys.public.as_deref() == Some(auth_header) + data.api_keys().master.as_deref() == Some(auth_header) + || data.api_keys().private.as_deref() == Some(auth_header) + || data.api_keys().public.as_deref() == Some(auth_header) } }; diff --git a/src/lib.rs b/src/lib.rs index 0f2aabccd..2c625588d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,7 +35,7 @@ pub fn create_app( .data(data.clone()) .app_data( web::JsonConfig::default() - .limit(data.http_payload_size_limit) + .limit(data.http_payload_size_limit()) .content_type(|_mime| true) // Accept all mime types .error_handler(|err, _req| payload_error_handler(err).into()), ) diff --git a/src/main.rs b/src/main.rs index e5b167cd0..7e50225be 100644 --- a/src/main.rs +++ b/src/main.rs @@ -147,7 +147,7 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { eprintln!(); - if data.api_keys.master.is_some() { + if data.api_keys().master.is_some() { eprintln!("A Master Key has been set. Requests to MeiliSearch won't be authorized unless you provide an authentication key."); } else { eprintln!("No master key found; The server will accept unidentified requests. \ diff --git a/src/updates/mod.rs b/src/updates/mod.rs index dd76ed1aa..30490ec43 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -3,7 +3,6 @@ mod settings; pub use settings::{Settings, Facets}; use std::io; -use std::path::Path; use std::sync::Arc; use anyhow::Result; @@ -41,6 +40,7 @@ enum UpdateMetaProgress { #[derive(Debug, Clone, Serialize)] #[serde(tag = "type")] +#[allow(dead_code)] enum UpdateStatus { Pending { update_id: u64, meta: M }, Progressing { update_id: u64, meta: P }, @@ -331,8 +331,8 @@ impl Handler for UpdateHandler { } impl UpdateQueue { - pub fn new>( - opt: Opt, + pub fn new( + opt: &Opt, indexes: Arc, ) -> Result { let (sender, _) = broadcast::channel(100); From 1a38bfd31fcf482f0b479fe3b6bfad809bb44ea7 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 23 Dec 2020 13:52:28 +0100 Subject: [PATCH 006/522] data add documents --- Cargo.lock | 41 ++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 5 ++++- src/data.rs | 40 ++++++++++++++++++++++++++++++++++++++-- src/routes/document.rs | 4 ++-- src/updates/mod.rs | 35 ++++++++++++++++------------------- 5 files changed, 100 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 610c83318..451b8efc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,6 +332,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite 0.1.11", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.42" @@ -1530,6 +1543,7 @@ dependencies = [ "actix-web", "anyhow", "assert-json-diff", + "async-compression", "byte-unit", "bytes 0.6.0", "chrono", @@ -1537,6 +1551,7 @@ dependencies = [ "env_logger 0.8.2", "flate2", "futures", + "futures-util", "grenad", "heed", "http", @@ -1545,6 +1560,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", + "memmap", "milli", "mime", "once_cell", @@ -1680,12 +1696,24 @@ dependencies = [ "kernel32-sys", "libc", "log", - "miow", + "miow 0.2.2", "net2", "slab", "winapi 0.2.8", ] +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio", + "miow 0.3.6", + "winapi 0.3.9", +] + [[package]] name = "mio-uds" version = "0.6.8" @@ -1709,6 +1737,16 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "miow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + [[package]] name = "near-proximity" version = "0.1.0" @@ -2895,6 +2933,7 @@ dependencies = [ "libc", "memchr", "mio", + "mio-named-pipes", "mio-uds", "num_cpus", "pin-project-lite 0.1.11", diff --git a/Cargo.toml b/Cargo.toml index cc25f0c8c..c6dd1db76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ actix-rt = "1" actix-service = "1.0.6" actix-web = { version = "3.3.2", features = ["rustls"] } anyhow = "1.0.36" +async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } bytes = "0.6.0" chrono = { version = "0.4.19", features = ["serde"] } @@ -49,10 +50,12 @@ slice-group-by = "0.2.6" structopt = "0.3.20" tar = "0.4.29" tempfile = "3.1.0" -tokio = "*" +tokio = { version = "*", features = ["full"] } ureq = { version = "1.5.1", default-features = false, features = ["tls"] } walkdir = "2.3.1" whoami = "1.0.0" +futures-util = "0.3.8" +memmap = "0.7.0" [dependencies.sentry] default-features = false diff --git a/src/data.rs b/src/data.rs index 621d8cf2b..c53042b8e 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,11 +1,15 @@ use std::ops::Deref; use std::sync::Arc; -use sha2::Digest; +use async_compression::tokio_02::write::GzipEncoder; +use futures_util::stream::StreamExt; +use tokio::io::AsyncWriteExt; use milli::Index; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use sha2::Digest; use crate::option::Opt; -use crate::updates::UpdateQueue; +use crate::updates::{UpdateQueue, UpdateMeta, UpdateStatus, UpdateMetaProgress}; #[derive(Clone)] pub struct Data { @@ -75,11 +79,43 @@ impl Data { Ok(Data { inner }) } + pub async fn add_documents( + &self, + method: IndexDocumentsMethod, + format: UpdateFormat, + mut stream: impl futures::Stream> + Unpin, + ) -> anyhow::Result> + where + B: Deref, + E: std::error::Error + Send + Sync + 'static, + { + let file = tokio::task::block_in_place(tempfile::tempfile)?; + let file = tokio::fs::File::from_std(file); + let mut encoder = GzipEncoder::new(file); + + while let Some(result) = stream.next().await { + let bytes = &*result?; + encoder.write_all(&bytes[..]).await?; + } + + encoder.shutdown().await?; + let mut file = encoder.into_inner(); + file.sync_all().await?; + let file = file.into_std().await; + let mmap = unsafe { memmap::Mmap::map(&file)? }; + + let meta = UpdateMeta::DocumentsAddition { method, format }; + let update_id = tokio::task::block_in_place(|| self.update_queue.register_update(&meta, &mmap[..]))?; + + Ok(UpdateStatus::Pending { update_id, meta }) + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize } + #[inline] pub fn api_keys(&self) -> &ApiKeys { &self.api_keys } diff --git a/src/routes/document.rs b/src/routes/document.rs index 3dfa36880..bdd9a8336 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -93,10 +93,10 @@ async fn update_multiple_documents( #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn add_documents( - _data: web::Data, + data: web::Data, _path: web::Path, _params: web::Query, - _body: web::Json>, + body: web::Json>, ) -> Result { todo!() } diff --git a/src/updates/mod.rs b/src/updates/mod.rs index 30490ec43..00faa4d85 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -4,6 +4,7 @@ pub use settings::{Settings, Facets}; use std::io; use std::sync::Arc; +use std::ops::Deref; use anyhow::Result; use flate2::read::GzDecoder; @@ -20,8 +21,8 @@ use crate::option::Opt; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] -enum UpdateMeta { - DocumentsAddition { method: String, format: String }, +pub enum UpdateMeta { + DocumentsAddition { method: IndexDocumentsMethod, format: UpdateFormat }, ClearDocuments, Settings(Settings), Facets(Facets), @@ -29,7 +30,7 @@ enum UpdateMeta { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] -enum UpdateMetaProgress { +pub enum UpdateMetaProgress { DocumentsAddition { step: usize, total_steps: usize, @@ -41,7 +42,7 @@ enum UpdateMetaProgress { #[derive(Debug, Clone, Serialize)] #[serde(tag = "type")] #[allow(dead_code)] -enum UpdateStatus { +pub enum UpdateStatus { Pending { update_id: u64, meta: M }, Progressing { update_id: u64, meta: P }, Processed { update_id: u64, meta: N }, @@ -53,6 +54,13 @@ pub struct UpdateQueue { inner: Arc>, } +impl Deref for UpdateQueue { + type Target = Arc>; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} #[derive(Debug, Clone, StructOpt)] pub struct IndexerOpts { @@ -164,27 +172,16 @@ impl UpdateHandler { fn update_documents( &self, - format: String, - method: String, + format: UpdateFormat, + method: IndexDocumentsMethod, content: &[u8], update_builder: UpdateBuilder, ) -> Result<()> { // We must use the write transaction of the update here. let mut wtxn = self.indexes.write_txn()?; let mut builder = update_builder.index_documents(&mut wtxn, &self.indexes); - - match format.as_str() { - "csv" => builder.update_format(UpdateFormat::Csv), - "json" => builder.update_format(UpdateFormat::Json), - "json-stream" => builder.update_format(UpdateFormat::JsonStream), - otherwise => panic!("invalid update format {:?}", otherwise), - }; - - match method.as_str() { - "replace" => builder.index_documents_method(IndexDocumentsMethod::ReplaceDocuments), - "update" => builder.index_documents_method(IndexDocumentsMethod::UpdateDocuments), - otherwise => panic!("invalid indexing method {:?}", otherwise), - }; + builder.update_format(format); + builder.index_documents_method(method); let gzipped = true; let reader = if gzipped { From 02ef1d41d74680178c2ba6dc5b9f619fc53032b3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 23 Dec 2020 16:12:37 +0100 Subject: [PATCH 007/522] route document add json --- Cargo.toml | 4 +-- src/data.rs | 15 +++++++---- src/routes/document.rs | 61 ++++++++++++++++++++++++++++++++++++++---- src/routes/mod.rs | 2 +- src/updates/mod.rs | 2 ++ 5 files changed, 71 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c6dd1db76..4395b2694 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ slice-group-by = "0.2.6" structopt = "0.3.20" tar = "0.4.29" tempfile = "3.1.0" -tokio = { version = "*", features = ["full"] } +tokio = { version = "0.2", features = ["full"] } ureq = { version = "1.5.1", default-features = false, features = ["tls"] } walkdir = "2.3.1" whoami = "1.0.0" @@ -68,7 +68,7 @@ version = "0.18.1" serde_url_params = "0.2.0" tempdir = "0.3.7" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } -tokio = { version = "0.2.18", features = ["macros", "time"] } +tokio = { version = "0.2", features = ["macros", "time"] } [features] default = ["sentry"] diff --git a/src/data.rs b/src/data.rs index c53042b8e..0702d7364 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,5 +1,6 @@ use std::ops::Deref; use std::sync::Arc; +use std::fs::create_dir_all; use async_compression::tokio_02::write::GzipEncoder; use futures_util::stream::StreamExt; @@ -27,7 +28,7 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { pub indexes: Arc, - pub update_queue: UpdateQueue, + pub update_queue: Arc, api_keys: ApiKeys, options: Opt, } @@ -60,10 +61,11 @@ impl Data { pub fn new(options: Opt) -> anyhow::Result { let db_size = options.max_mdb_size.get_bytes() as usize; let path = options.db_path.join("main"); + create_dir_all(&path)?; let indexes = Index::new(&path, Some(db_size))?; let indexes = Arc::new(indexes); - let update_queue = UpdateQueue::new(&options, indexes.clone())?; + let update_queue = Arc::new(UpdateQueue::new(&options, indexes.clone())?); let mut api_keys = ApiKeys { master: options.clone().master_key, @@ -89,8 +91,8 @@ impl Data { B: Deref, E: std::error::Error + Send + Sync + 'static, { - let file = tokio::task::block_in_place(tempfile::tempfile)?; - let file = tokio::fs::File::from_std(file); + let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; + let file = tokio::fs::File::from_std(file?); let mut encoder = GzipEncoder::new(file); while let Some(result) = stream.next().await { @@ -105,7 +107,10 @@ impl Data { let mmap = unsafe { memmap::Mmap::map(&file)? }; let meta = UpdateMeta::DocumentsAddition { method, format }; - let update_id = tokio::task::block_in_place(|| self.update_queue.register_update(&meta, &mmap[..]))?; + + let queue = self.update_queue.clone(); + let meta_cloned = meta.clone(); + let update_id = tokio::task::spawn_blocking(move || queue.register_update(&meta_cloned, &mmap[..])).await??; Ok(UpdateStatus::Pending { update_id, meta }) } diff --git a/src/routes/document.rs b/src/routes/document.rs index bdd9a8336..c24c01af8 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -1,14 +1,31 @@ +use actix_web::web::Payload; use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use indexmap::IndexMap; -use serde_json::Value; +use log::error; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; +use serde_json::Value; use crate::Data; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; +macro_rules! guard_content_type { + ($fn_name:ident, $guard_value:literal) => { + fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { + if let Some(content_type) = head.headers.get("Content-Type") { + content_type.to_str().map(|v| v.contains($guard_value)).unwrap_or(false) + } else { + false + } + } + }; +} + +guard_content_type!(guard_json, "application/json"); + type Document = IndexMap; #[derive(Deserialize)] @@ -21,7 +38,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(get_document) .service(delete_document) .service(get_all_documents) - .service(add_documents) + .service(add_documents_json) .service(update_documents) .service(delete_documents) .service(clear_all_documents); @@ -91,12 +108,46 @@ async fn update_multiple_documents( todo!() } -#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] -async fn add_documents( +/// Route used when the payload type is "application/json" +#[post( + "/indexes/{index_uid}/documents", + wrap = "Authentication::Private", + guard = "guard_json" +)] +async fn add_documents_json( data: web::Data, + path: web::Path, + params: web::Query, + body: Payload, +) -> Result { + let addition_result = data + .add_documents( + IndexDocumentsMethod::UpdateDocuments, + UpdateFormat::Json, + body + ).await; + + match addition_result { + Ok(update) => { + let value = serde_json::to_string(&update).unwrap(); + let response = HttpResponse::Ok().body(value); + Ok(response) + } + Err(e) => { + error!("{}", e); + todo!() + } + } +} + + +/// Default route for addign documents, this should return an error en redirect to the docuentation +#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] +async fn add_documents_default( + _data: web::Data, _path: web::Path, _params: web::Query, - body: web::Json>, + _body: web::Json>, ) -> Result { todo!() } diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 71cb2a8d8..4b0813067 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -14,7 +14,7 @@ pub mod synonym; #[derive(Deserialize)] pub struct IndexParam { - _index_uid: String, + index_uid: String, } #[derive(Serialize)] diff --git a/src/updates/mod.rs b/src/updates/mod.rs index 00faa4d85..f08439c74 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -5,6 +5,7 @@ pub use settings::{Settings, Facets}; use std::io; use std::sync::Arc; use std::ops::Deref; +use std::fs::create_dir_all; use anyhow::Result; use flate2::read::GzDecoder; @@ -336,6 +337,7 @@ impl UpdateQueue { let handler = UpdateHandler::new(&opt.indexer_options, indexes, sender)?; let size = opt.max_udb_size.get_bytes() as usize; let path = opt.db_path.join("updates.mdb"); + create_dir_all(&path)?; let inner = UpdateStore::open( Some(size), path, From 0cd9e62fc6da00242f03af7aaf71f99cc379eeaa Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 24 Dec 2020 12:58:34 +0100 Subject: [PATCH 008/522] search first iteration --- Cargo.lock | 139 +++++++++++++++++++++++++++++++++++- Cargo.toml | 6 +- src/data.rs | 157 ++++++++++++++++++++++++++++++++++++++++- src/routes/document.rs | 3 +- src/routes/search.rs | 68 ++++++++---------- 5 files changed, 329 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 451b8efc0..c5cea3936 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,6 +299,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" + [[package]] name = "aho-corasick" version = "0.7.15" @@ -574,6 +580,15 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cedarwood" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963e82c7b94163808ca3a452608d260b64ba5bc7b5653b4af1af59887899f48d" +dependencies = [ + "smallvec", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -586,6 +601,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "character_converter" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e48477ece09d6a21c033cb604968524a37782532727055d6f6faafac1781e5c" +dependencies = [ + "bincode", +] + [[package]] name = "chrono" version = "0.4.19" @@ -645,6 +669,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" +[[package]] +name = "cow-utils" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -768,6 +798,12 @@ dependencies = [ "syn", ] +[[package]] +name = "deunicode" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" + [[package]] name = "digest" version = "0.8.1" @@ -1127,6 +1163,16 @@ dependencies = [ "tracing-futures", ] +[[package]] +name = "hashbrown" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" +dependencies = [ + "ahash", + "autocfg", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -1326,7 +1372,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.9.1", "serde", ] @@ -1402,6 +1448,21 @@ dependencies = [ "libc", ] +[[package]] +name = "jieba-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34fbdeee8786790f4a99fa30ff5c5f88aa5183f7583693e3788d17fc8a48f33a" +dependencies = [ + "cedarwood", + "fxhash", + "hashbrown 0.9.1", + "lazy_static", + "phf", + "phf_codegen", + "regex", +] + [[package]] name = "jobserver" version = "0.1.21" @@ -1550,6 +1611,7 @@ dependencies = [ "crossbeam-channel", "env_logger 0.8.2", "flate2", + "fst", "futures", "futures-util", "grenad", @@ -1560,6 +1622,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", + "meilisearch-tokenizer", "memmap", "milli", "mime", @@ -1587,6 +1650,22 @@ dependencies = [ "whoami", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.1.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#147b6154b1b34cb8f5da2df6a416b7da191bc850" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "memchr" version = "2.3.4" @@ -1634,6 +1713,7 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", + "meilisearch-tokenizer", "memmap", "near-proximity", "num-traits", @@ -1938,6 +2018,44 @@ dependencies = [ "sha-1 0.8.2", ] +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared", + "rand 0.7.3", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "0.4.27" @@ -2101,6 +2219,7 @@ dependencies = [ "rand_chacha", "rand_core 0.5.1", "rand_hc", + "rand_pcg", ] [[package]] @@ -2146,6 +2265,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rand_xoshiro" version = "0.4.0" @@ -3342,6 +3470,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "whatlang" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075" +dependencies = [ + "hashbrown 0.7.2", +] + [[package]] name = "whoami" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 4395b2694..fb8531a96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,9 @@ chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.0" env_logger = "0.8.2" flate2 = "1.0.19" +fst = "0.4.5" futures = "0.3.7" +futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } heed = "0.10.6" http = "0.2.1" @@ -34,6 +36,8 @@ indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } +memmap = "0.7.0" milli = { path = "../milli" } mime = "0.3.16" once_cell = "1.5.2" @@ -54,8 +58,6 @@ tokio = { version = "0.2", features = ["full"] } ureq = { version = "1.5.1", default-features = false, features = ["tls"] } walkdir = "2.3.1" whoami = "1.0.0" -futures-util = "0.3.8" -memmap = "0.7.0" [dependencies.sentry] default-features = false diff --git a/src/data.rs b/src/data.rs index 0702d7364..48826a44f 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,17 +1,53 @@ +use std::borrow::Cow; +use std::collections::HashSet; +use std::fs::create_dir_all; +use std::mem; use std::ops::Deref; use std::sync::Arc; -use std::fs::create_dir_all; +use std::time::Instant; use async_compression::tokio_02::write::GzipEncoder; use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; -use milli::Index; +use milli::{Index, SearchResult as Results, obkv_to_json}; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use sha2::Digest; +use serde_json::{Value, Map}; +use serde::{Deserialize, Serialize}; +use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; use crate::option::Opt; use crate::updates::{UpdateQueue, UpdateMeta, UpdateStatus, UpdateMetaProgress}; +const DEFAULT_SEARCH_LIMIT: usize = 20; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchQuery { + q: Option, + offset: Option, + limit: Option, + attributes_to_retrieve: Option>, + attributes_to_crop: Option>, + crop_length: Option, + attributes_to_highlight: Option>, + filters: Option, + matches: Option, + facet_filters: Option, + facets_distribution: Option>, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchResult { + hits: Vec>, + nb_hits: usize, + query: String, + limit: usize, + offset: usize, + processing_time_ms: u128, +} + #[derive(Clone)] pub struct Data { inner: Arc, @@ -81,8 +117,9 @@ impl Data { Ok(Data { inner }) } - pub async fn add_documents( + pub async fn add_documents( &self, + _index: S, method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, @@ -90,6 +127,7 @@ impl Data { where B: Deref, E: std::error::Error + Send + Sync + 'static, + S: AsRef, { let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; let file = tokio::fs::File::from_std(file?); @@ -115,6 +153,60 @@ impl Data { Ok(UpdateStatus::Pending { update_id, meta }) } + pub fn search>(&self, _index: S, search_query: SearchQuery) -> anyhow::Result { + let start = Instant::now(); + let index = &self.indexes; + let rtxn = index.read_txn()?; + + let mut search = index.search(&rtxn); + if let Some(query) = &search_query.q { + search.query(query); + } + + if let Some(offset) = search_query.offset { + search.offset(offset); + } + + let limit = search_query.limit.unwrap_or(DEFAULT_SEARCH_LIMIT); + search.limit(limit); + + let Results { found_words, documents_ids, nb_hits, .. } = search.execute().unwrap(); + + let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields = match index.displayed_fields(&rtxn).unwrap() { + Some(fields) => Cow::Borrowed(fields), + None => Cow::Owned(fields_ids_map.iter().map(|(id, _)| id).collect()), + }; + + let attributes_to_highlight = match search_query.attributes_to_highlight { + Some(fields) => fields.iter().map(ToOwned::to_owned).collect(), + None => HashSet::new(), + }; + + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new(&stop_words); + let mut documents = Vec::new(); + for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { + let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); + highlighter.highlight_record(&mut object, &found_words, &attributes_to_highlight); + documents.push(object); + } + + let processing_time_ms = start.elapsed().as_millis(); + + let result = SearchResult { + hits: documents, + nb_hits, + query: search_query.q.unwrap_or_default(), + offset: search_query.offset.unwrap_or(0), + limit, + processing_time_ms, + }; + + Ok(result) + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize @@ -125,3 +217,62 @@ impl Data { &self.api_keys } } + +struct Highlighter<'a, A> { + analyzer: Analyzer<'a, A>, +} + +impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { + fn new(stop_words: &'a fst::Set) -> Self { + let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); + Self { analyzer } + } + + fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + match value { + Value::Null => Value::Null, + Value::Bool(boolean) => Value::Bool(boolean), + Value::Number(number) => Value::Number(number), + Value::String(old_string) => { + let mut string = String::new(); + let analyzed = self.analyzer.analyze(&old_string); + for (word, token) in analyzed.reconstruct() { + if token.is_word() { + let to_highlight = words_to_highlight.contains(token.text()); + if to_highlight { string.push_str("") } + string.push_str(word); + if to_highlight { string.push_str("") } + } else { + string.push_str(word); + } + } + Value::String(string) + }, + Value::Array(values) => { + Value::Array(values.into_iter() + .map(|v| self.highlight_value(v, words_to_highlight)) + .collect()) + }, + Value::Object(object) => { + Value::Object(object.into_iter() + .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) + .collect()) + }, + } + } + + fn highlight_record( + &self, + object: &mut Map, + words_to_highlight: &HashSet, + attributes_to_highlight: &HashSet, + ) { + // TODO do we need to create a string for element that are not and needs to be highlight? + for (key, value) in object.iter_mut() { + if attributes_to_highlight.contains(key) { + let old_value = mem::take(value); + *value = self.highlight_value(old_value, words_to_highlight); + } + } + } +} diff --git a/src/routes/document.rs b/src/routes/document.rs index c24c01af8..6c5f93991 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -117,11 +117,12 @@ async fn update_multiple_documents( async fn add_documents_json( data: web::Data, path: web::Path, - params: web::Query, + _params: web::Query, body: Payload, ) -> Result { let addition_result = data .add_documents( + &path.index_uid, IndexDocumentsMethod::UpdateDocuments, UpdateFormat::Json, body diff --git a/src/routes/search.rs b/src/routes/search.rs index 2c510e0d4..967065687 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -1,31 +1,31 @@ use actix_web::{get, post, web, HttpResponse}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; +use log::error; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; use crate::Data; +use crate::data::SearchQuery; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(search_with_post).service(search_with_url_query); } -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct SearchQuery { - q: Option, - offset: Option, - limit: Option, - attributes_to_retrieve: Option, - attributes_to_crop: Option, - crop_length: Option, - attributes_to_highlight: Option, - filters: Option, - matches: Option, - facet_filters: Option, - facets_distribution: Option, -} +//#[derive(Serialize, Deserialize)] +//#[serde(rename_all = "camelCase", deny_unknown_fields)] +//pub struct SearchQuery { + //q: Option, + //offset: Option, + //limit: Option, + //attributes_to_retrieve: Option, + //attributes_to_crop: Option, + //crop_length: Option, + //attributes_to_highlight: Option, + //filters: Option, + //matches: Option, + //facet_filters: Option, + //facets_distribution: Option, +//} #[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")] async fn search_with_url_query( @@ -36,27 +36,21 @@ async fn search_with_url_query( todo!() } -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct SearchQueryPost { - _q: Option, - _offset: Option, - _limit: Option, - _attributes_to_retrieve: Option>, - _attributes_to_crop: Option>, - _crop_length: Option, - _attributes_to_highlight: Option>, - _filters: Option, - _matches: Option, - _facet_filters: Option, - _facets_distribution: Option>, -} - #[post("/indexes/{index_uid}/search", wrap = "Authentication::Public")] async fn search_with_post( - _data: web::Data, - _path: web::Path, - _params: web::Json, + data: web::Data, + path: web::Path, + params: web::Json, ) -> Result { - todo!() + let search_result = data.search(&path.index_uid, params.into_inner()); + match search_result { + Ok(docs) => { + let docs = serde_json::to_string(&docs).unwrap(); + Ok(HttpResponse::Ok().body(docs)) + } + Err(e) => { + error!("{}", e); + todo!() + } + } } From 54861335a0f56c46e1c20cac7df85d617501d668 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 29 Dec 2020 11:11:06 +0100 Subject: [PATCH 009/522] retrieve update status --- Cargo.lock | 52 ++++++- src/data/mod.rs | 94 +++++++++++++ src/{data.rs => data/search.rs} | 240 ++++++++------------------------ src/data/updates.rs | 53 +++++++ src/routes/index.rs | 24 +++- src/updates/mod.rs | 112 +++++---------- 6 files changed, 308 insertions(+), 267 deletions(-) create mode 100644 src/data/mod.rs rename src/{data.rs => data/search.rs} (56%) create mode 100644 src/data/updates.rs diff --git a/Cargo.lock b/Cargo.lock index c5cea3936..13f74b82d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1188,6 +1188,22 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heed" +version = "0.10.6" +dependencies = [ + "byteorder", + "heed-traits 0.7.0", + "heed-types 0.7.2", + "libc", + "lmdb-rkv-sys", + "once_cell", + "page_size", + "synchronoise", + "url", + "zerocopy", +] + [[package]] name = "heed" version = "0.10.6" @@ -1195,8 +1211,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" dependencies = [ "byteorder", - "heed-traits", - "heed-types", + "heed-traits 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "heed-types 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc", "lmdb-rkv-sys", "once_cell", @@ -1207,12 +1223,27 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "heed-traits" +version = "0.7.0" + [[package]] name = "heed-traits" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" +[[package]] +name = "heed-types" +version = "0.7.2" +dependencies = [ + "bincode", + "heed-traits 0.7.0", + "serde", + "serde_json", + "zerocopy", +] + [[package]] name = "heed-types" version = "0.7.2" @@ -1220,7 +1251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" dependencies = [ "bincode", - "heed-traits", + "heed-traits 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde", "serde_json", "zerocopy", @@ -1615,7 +1646,7 @@ dependencies = [ "futures", "futures-util", "grenad", - "heed", + "heed 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "http", "indexmap", "jemallocator", @@ -1699,6 +1730,7 @@ dependencies = [ "bstr", "byte-unit", "byteorder", + "chrono", "crossbeam-channel", "csv", "either", @@ -1706,7 +1738,7 @@ dependencies = [ "fst", "fxhash", "grenad", - "heed", + "heed 0.10.6", "human_format", "itertools", "jemallocator", @@ -1728,6 +1760,7 @@ dependencies = [ "roaring", "serde", "serde_json", + "serde_millis", "slice-group-by", "smallstr", "smallvec", @@ -2594,6 +2627,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_millis" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e2dc780ca5ee2c369d1d01d100270203c4ff923d2a4264812d723766434d00" +dependencies = [ + "serde", +] + [[package]] name = "serde_qs" version = "0.8.2" diff --git a/src/data/mod.rs b/src/data/mod.rs new file mode 100644 index 000000000..a699c9d20 --- /dev/null +++ b/src/data/mod.rs @@ -0,0 +1,94 @@ +mod search; +mod updates; + +pub use search::{SearchQuery, SearchResult}; + +use std::fs::create_dir_all; +use std::ops::Deref; +use std::sync::Arc; + +use milli::Index; +use sha2::Digest; + +use crate::option::Opt; +use crate::updates::UpdateQueue; + +#[derive(Clone)] +pub struct Data { + inner: Arc, +} + +impl Deref for Data { + type Target = DataInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[derive(Clone)] +pub struct DataInner { + pub indexes: Arc, + pub update_queue: Arc, + api_keys: ApiKeys, + options: Opt, +} + +#[derive(Clone)] +pub struct ApiKeys { + pub public: Option, + pub private: Option, + pub master: Option, +} + +impl ApiKeys { + pub fn generate_missing_api_keys(&mut self) { + if let Some(master_key) = &self.master { + if self.private.is_none() { + let key = format!("{}-private", master_key); + let sha = sha2::Sha256::digest(key.as_bytes()); + self.private = Some(format!("{:x}", sha)); + } + if self.public.is_none() { + let key = format!("{}-public", master_key); + let sha = sha2::Sha256::digest(key.as_bytes()); + self.public = Some(format!("{:x}", sha)); + } + } + } +} + +impl Data { + pub fn new(options: Opt) -> anyhow::Result { + let db_size = options.max_mdb_size.get_bytes() as usize; + let path = options.db_path.join("main"); + create_dir_all(&path)?; + let indexes = Index::new(&path, Some(db_size))?; + let indexes = Arc::new(indexes); + + let update_queue = Arc::new(UpdateQueue::new(&options, indexes.clone())?); + + let mut api_keys = ApiKeys { + master: options.clone().master_key, + private: None, + public: None, + }; + + api_keys.generate_missing_api_keys(); + + let inner = DataInner { indexes, options, update_queue, api_keys }; + let inner = Arc::new(inner); + + Ok(Data { inner }) + } + + #[inline] + pub fn http_payload_size_limit(&self) -> usize { + self.options.http_payload_size_limit.get_bytes() as usize + } + + #[inline] + pub fn api_keys(&self) -> &ApiKeys { + &self.api_keys + } +} diff --git a/src/data.rs b/src/data/search.rs similarity index 56% rename from src/data.rs rename to src/data/search.rs index 48826a44f..bd22a959b 100644 --- a/src/data.rs +++ b/src/data/search.rs @@ -1,28 +1,20 @@ use std::borrow::Cow; use std::collections::HashSet; -use std::fs::create_dir_all; use std::mem; -use std::ops::Deref; -use std::sync::Arc; use std::time::Instant; -use async_compression::tokio_02::write::GzipEncoder; -use futures_util::stream::StreamExt; -use tokio::io::AsyncWriteExt; -use milli::{Index, SearchResult as Results, obkv_to_json}; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use sha2::Digest; use serde_json::{Value, Map}; use serde::{Deserialize, Serialize}; +use milli::{SearchResult as Results, obkv_to_json}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use crate::option::Opt; -use crate::updates::{UpdateQueue, UpdateMeta, UpdateStatus, UpdateMetaProgress}; +use super::Data; const DEFAULT_SEARCH_LIMIT: usize = 20; #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] +#[allow(dead_code)] pub struct SearchQuery { q: Option, offset: Option, @@ -48,176 +40,6 @@ pub struct SearchResult { processing_time_ms: u128, } -#[derive(Clone)] -pub struct Data { - inner: Arc, -} - -impl Deref for Data { - type Target = DataInner; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -#[derive(Clone)] -pub struct DataInner { - pub indexes: Arc, - pub update_queue: Arc, - api_keys: ApiKeys, - options: Opt, -} - -#[derive(Clone)] -pub struct ApiKeys { - pub public: Option, - pub private: Option, - pub master: Option, -} - -impl ApiKeys { - pub fn generate_missing_api_keys(&mut self) { - if let Some(master_key) = &self.master { - if self.private.is_none() { - let key = format!("{}-private", master_key); - let sha = sha2::Sha256::digest(key.as_bytes()); - self.private = Some(format!("{:x}", sha)); - } - if self.public.is_none() { - let key = format!("{}-public", master_key); - let sha = sha2::Sha256::digest(key.as_bytes()); - self.public = Some(format!("{:x}", sha)); - } - } - } -} - -impl Data { - pub fn new(options: Opt) -> anyhow::Result { - let db_size = options.max_mdb_size.get_bytes() as usize; - let path = options.db_path.join("main"); - create_dir_all(&path)?; - let indexes = Index::new(&path, Some(db_size))?; - let indexes = Arc::new(indexes); - - let update_queue = Arc::new(UpdateQueue::new(&options, indexes.clone())?); - - let mut api_keys = ApiKeys { - master: options.clone().master_key, - private: None, - public: None, - }; - - api_keys.generate_missing_api_keys(); - - let inner = DataInner { indexes, options, update_queue, api_keys }; - let inner = Arc::new(inner); - - Ok(Data { inner }) - } - - pub async fn add_documents( - &self, - _index: S, - method: IndexDocumentsMethod, - format: UpdateFormat, - mut stream: impl futures::Stream> + Unpin, - ) -> anyhow::Result> - where - B: Deref, - E: std::error::Error + Send + Sync + 'static, - S: AsRef, - { - let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; - let file = tokio::fs::File::from_std(file?); - let mut encoder = GzipEncoder::new(file); - - while let Some(result) = stream.next().await { - let bytes = &*result?; - encoder.write_all(&bytes[..]).await?; - } - - encoder.shutdown().await?; - let mut file = encoder.into_inner(); - file.sync_all().await?; - let file = file.into_std().await; - let mmap = unsafe { memmap::Mmap::map(&file)? }; - - let meta = UpdateMeta::DocumentsAddition { method, format }; - - let queue = self.update_queue.clone(); - let meta_cloned = meta.clone(); - let update_id = tokio::task::spawn_blocking(move || queue.register_update(&meta_cloned, &mmap[..])).await??; - - Ok(UpdateStatus::Pending { update_id, meta }) - } - - pub fn search>(&self, _index: S, search_query: SearchQuery) -> anyhow::Result { - let start = Instant::now(); - let index = &self.indexes; - let rtxn = index.read_txn()?; - - let mut search = index.search(&rtxn); - if let Some(query) = &search_query.q { - search.query(query); - } - - if let Some(offset) = search_query.offset { - search.offset(offset); - } - - let limit = search_query.limit.unwrap_or(DEFAULT_SEARCH_LIMIT); - search.limit(limit); - - let Results { found_words, documents_ids, nb_hits, .. } = search.execute().unwrap(); - - let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); - - let displayed_fields = match index.displayed_fields(&rtxn).unwrap() { - Some(fields) => Cow::Borrowed(fields), - None => Cow::Owned(fields_ids_map.iter().map(|(id, _)| id).collect()), - }; - - let attributes_to_highlight = match search_query.attributes_to_highlight { - Some(fields) => fields.iter().map(ToOwned::to_owned).collect(), - None => HashSet::new(), - }; - - let stop_words = fst::Set::default(); - let highlighter = Highlighter::new(&stop_words); - let mut documents = Vec::new(); - for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { - let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); - highlighter.highlight_record(&mut object, &found_words, &attributes_to_highlight); - documents.push(object); - } - - let processing_time_ms = start.elapsed().as_millis(); - - let result = SearchResult { - hits: documents, - nb_hits, - query: search_query.q.unwrap_or_default(), - offset: search_query.offset.unwrap_or(0), - limit, - processing_time_ms, - }; - - Ok(result) - } - - #[inline] - pub fn http_payload_size_limit(&self) -> usize { - self.options.http_payload_size_limit.get_bytes() as usize - } - - #[inline] - pub fn api_keys(&self) -> &ApiKeys { - &self.api_keys - } -} - struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, } @@ -276,3 +98,59 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } } } + +impl Data { + pub fn search>(&self, _index: S, search_query: SearchQuery) -> anyhow::Result { + let start = Instant::now(); + let index = &self.indexes; + let rtxn = index.read_txn()?; + + let mut search = index.search(&rtxn); + if let Some(query) = &search_query.q { + search.query(query); + } + + if let Some(offset) = search_query.offset { + search.offset(offset); + } + + let limit = search_query.limit.unwrap_or(DEFAULT_SEARCH_LIMIT); + search.limit(limit); + + let Results { found_words, documents_ids, nb_hits, .. } = search.execute().unwrap(); + + let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields = match index.displayed_fields(&rtxn).unwrap() { + Some(fields) => Cow::Borrowed(fields), + None => Cow::Owned(fields_ids_map.iter().map(|(id, _)| id).collect()), + }; + + let attributes_to_highlight = match search_query.attributes_to_highlight { + Some(fields) => fields.iter().map(ToOwned::to_owned).collect(), + None => HashSet::new(), + }; + + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new(&stop_words); + let mut documents = Vec::new(); + for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { + let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); + highlighter.highlight_record(&mut object, &found_words, &attributes_to_highlight); + documents.push(object); + } + + let processing_time_ms = start.elapsed().as_millis(); + + let result = SearchResult { + hits: documents, + nb_hits, + query: search_query.q.unwrap_or_default(), + offset: search_query.offset.unwrap_or(0), + limit, + processing_time_ms, + }; + + Ok(result) + } +} diff --git a/src/data/updates.rs b/src/data/updates.rs new file mode 100644 index 000000000..d12f271c8 --- /dev/null +++ b/src/data/updates.rs @@ -0,0 +1,53 @@ +use std::ops::Deref; + +use async_compression::tokio_02::write::GzipEncoder; +use futures_util::stream::StreamExt; +use tokio::io::AsyncWriteExt; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use milli::update_store::UpdateStatus; + +use super::Data; +use crate::updates::UpdateMeta; + +impl Data { + pub async fn add_documents( + &self, + _index: S, + method: IndexDocumentsMethod, + format: UpdateFormat, + mut stream: impl futures::Stream> + Unpin, + ) -> anyhow::Result> + where + B: Deref, + E: std::error::Error + Send + Sync + 'static, + S: AsRef, + { + let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; + let file = tokio::fs::File::from_std(file?); + let mut encoder = GzipEncoder::new(file); + + while let Some(result) = stream.next().await { + let bytes = &*result?; + encoder.write_all(&bytes[..]).await?; + } + + encoder.shutdown().await?; + let mut file = encoder.into_inner(); + file.sync_all().await?; + let file = file.into_std().await; + let mmap = unsafe { memmap::Mmap::map(&file)? }; + + let meta = UpdateMeta::DocumentsAddition { method, format }; + + let queue = self.update_queue.clone(); + let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &mmap[..])).await??; + + Ok(update.into()) + } + + + #[inline] + pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result>> { + self.update_queue.get_update_status(uid) + } +} diff --git a/src/routes/index.rs b/src/routes/index.rs index 36860f34f..eb961a8dd 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -1,6 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; +use log::error; use serde::{Deserialize, Serialize}; use crate::Data; @@ -93,8 +94,8 @@ async fn delete_index( #[derive(Deserialize)] struct UpdateParam { - _index_uid: String, - _update_id: u64, + index_uid: String, + update_id: u64, } #[get( @@ -102,10 +103,23 @@ struct UpdateParam { wrap = "Authentication::Private" )] async fn get_update_status( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() + let result = data.get_update_status(&path.index_uid, path.update_id); + match result { + Ok(Some(meta)) => { + let json = serde_json::to_string(&meta).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Ok(None) => { + todo!() + } + Err(e) => { + error!("{}", e); + todo!() + } + } } #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] diff --git a/src/updates/mod.rs b/src/updates/mod.rs index f08439c74..cd8052108 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -8,14 +8,15 @@ use std::ops::Deref; use std::fs::create_dir_all; use anyhow::Result; +use byte_unit::Byte; use flate2::read::GzDecoder; use grenad::CompressionType; -use byte_unit::Byte; -use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod, UpdateIndexingStep::*}; -use milli::{UpdateStore, UpdateHandler as Handler, Index}; +use log::info; +use milli::Index; +use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod }; +use milli::update_store::{UpdateStore, UpdateHandler as Handler, UpdateStatus, Processing, Processed, Failed}; use rayon::ThreadPool; use serde::{Serialize, Deserialize}; -use tokio::sync::broadcast; use structopt::StructOpt; use crate::option::Opt; @@ -40,23 +41,13 @@ pub enum UpdateMetaProgress { }, } -#[derive(Debug, Clone, Serialize)] -#[serde(tag = "type")] -#[allow(dead_code)] -pub enum UpdateStatus { - Pending { update_id: u64, meta: M }, - Progressing { update_id: u64, meta: P }, - Processed { update_id: u64, meta: N }, - Aborted { update_id: u64, meta: M }, -} - #[derive(Clone)] pub struct UpdateQueue { - inner: Arc>, + inner: Arc>, } impl Deref for UpdateQueue { - type Target = Arc>; + type Target = Arc>; fn deref(&self) -> &Self::Target { &self.inner @@ -115,8 +106,6 @@ pub struct IndexerOpts { pub indexing_jobs: Option, } -type UpdateSender = broadcast::Sender>; - struct UpdateHandler { indexes: Arc, max_nb_chunks: Option, @@ -127,14 +116,12 @@ struct UpdateHandler { linked_hash_map_size: usize, chunk_compression_type: CompressionType, chunk_fusing_shrink_size: u64, - update_status_sender: UpdateSender, } impl UpdateHandler { fn new( opt: &IndexerOpts, indexes: Arc, - update_status_sender: UpdateSender, ) -> Result { let thread_pool = rayon::ThreadPoolBuilder::new() .num_threads(opt.indexing_jobs.unwrap_or(0)) @@ -149,7 +136,6 @@ impl UpdateHandler { linked_hash_map_size: opt.linked_hash_map_size, chunk_compression_type: opt.chunk_compression_type, chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), - update_status_sender, }) } @@ -191,23 +177,7 @@ impl UpdateHandler { Box::new(content) as Box }; - let result = builder.execute(reader, |indexing_step, update_id| { - let (current, total) = match indexing_step { - TransformFromUserIntoGenericFormat { documents_seen } => (documents_seen, None), - ComputeIdsAndMergeDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)), - IndexDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)), - MergeDataIntoFinalDatabase { databases_seen, total_databases } => (databases_seen, Some(total_databases)), - }; - let _ = self.update_status_sender.send(UpdateStatus::Progressing { - update_id, - meta: UpdateMetaProgress::DocumentsAddition { - step: indexing_step.step(), - total_steps: indexing_step.number_of_steps(), - current, - total, - } - }); - }); + let result = builder.execute(reader, |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); match result { Ok(()) => wtxn.commit().map_err(Into::into), @@ -226,57 +196,41 @@ impl UpdateHandler { } } - fn update_settings(&self, settings: Settings, update_builder: UpdateBuilder) -> Result<()> { + fn update_settings(&self, settings: &Settings, update_builder: UpdateBuilder) -> Result<()> { // We must use the write transaction of the update here. let mut wtxn = self.indexes.write_txn()?; let mut builder = update_builder.settings(&mut wtxn, &self.indexes); // We transpose the settings JSON struct into a real setting update. - if let Some(names) = settings.searchable_attributes { + if let Some(ref names) = settings.searchable_attributes { match names { - Some(names) => builder.set_searchable_fields(names), + Some(names) => builder.set_searchable_fields(&names), None => builder.reset_searchable_fields(), } } // We transpose the settings JSON struct into a real setting update. - if let Some(names) = settings.displayed_attributes { + if let Some(ref names) = settings.displayed_attributes { match names { - Some(names) => builder.set_displayed_fields(names), + Some(names) => builder.set_displayed_fields(&names), None => builder.reset_displayed_fields(), } } // We transpose the settings JSON struct into a real setting update. - if let Some(facet_types) = settings.faceted_attributes { - builder.set_faceted_fields(facet_types); + if let Some(ref facet_types) = settings.faceted_attributes { + builder.set_faceted_fields(&facet_types); } // We transpose the settings JSON struct into a real setting update. - if let Some(criteria) = settings.criteria { + if let Some(ref criteria) = settings.criteria { match criteria { - Some(criteria) => builder.set_criteria(criteria), + Some(criteria) => builder.set_criteria(&criteria), None => builder.reset_criteria(), } } - let result = builder.execute(|indexing_step, update_id| { - let (current, total) = match indexing_step { - TransformFromUserIntoGenericFormat { documents_seen } => (documents_seen, None), - ComputeIdsAndMergeDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)), - IndexDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)), - MergeDataIntoFinalDatabase { databases_seen, total_databases } => (databases_seen, Some(total_databases)), - }; - let _ = self.update_status_sender.send(UpdateStatus::Progressing { - update_id, - meta: UpdateMetaProgress::DocumentsAddition { - step: indexing_step.step(), - total_steps: indexing_step.number_of_steps(), - current, - total, - } - }); - }); + let result = builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); match result { Ok(_count) => wtxn.commit().map_err(Into::into), @@ -284,7 +238,7 @@ impl UpdateHandler { } } - fn update_facets(&self, levels: Facets, update_builder: UpdateBuilder) -> Result<()> { + fn update_facets(&self, levels: &Facets, update_builder: UpdateBuilder) -> Result<()> { // We must use the write transaction of the update here. let mut wtxn = self.indexes.write_txn()?; let mut builder = update_builder.facets(&mut wtxn, &self.indexes); @@ -301,28 +255,30 @@ impl UpdateHandler { } } -impl Handler for UpdateHandler { - fn handle_update(&mut self, update_id: u64, meta: UpdateMeta, content: &[u8]) -> heed::Result { +impl Handler for UpdateHandler { + fn handle_update( + &mut self, + update_id: u64, + meta: Processing, + content: &[u8] + ) -> Result, Failed> { use UpdateMeta::*; let update_builder = self.update_buidler(update_id); - let result: anyhow::Result<()> = match meta { - DocumentsAddition { method, format } => { - self.update_documents(format, method, content, update_builder) - }, + let result: anyhow::Result<()> = match meta.meta() { + DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), ClearDocuments => self.clear_documents(update_builder), Settings(settings) => self.update_settings(settings, update_builder), Facets(levels) => self.update_facets(levels, update_builder), }; - let meta = match result { + let new_meta = match result { Ok(()) => format!("valid update content"), Err(e) => format!("error while processing update content: {:?}", e), }; - let processed = UpdateStatus::Processed { update_id, meta: meta.clone() }; - let _ = self.update_status_sender.send(processed); + let meta = meta.process(new_meta); Ok(meta) } @@ -333,8 +289,7 @@ impl UpdateQueue { opt: &Opt, indexes: Arc, ) -> Result { - let (sender, _) = broadcast::channel(100); - let handler = UpdateHandler::new(&opt.indexer_options, indexes, sender)?; + let handler = UpdateHandler::new(&opt.indexer_options, indexes)?; let size = opt.max_udb_size.get_bytes() as usize; let path = opt.db_path.join("updates.mdb"); create_dir_all(&path)?; @@ -345,4 +300,9 @@ impl UpdateQueue { )?; Ok(Self { inner }) } + + #[inline] + pub fn get_update_status(&self, update_id: u64) -> Result>> { + Ok(self.inner.meta(update_id)?) + } } From d9dc2036a7bf9de3bbecb41bed97cb0575c85455 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 30 Dec 2020 18:44:33 +0100 Subject: [PATCH 010/522] support error & return document count on addition --- Cargo.lock | 41 ++++--------------------------- src/data/updates.rs | 4 +-- src/updates/mod.rs | 60 ++++++++++++++++++++++++++++----------------- 3 files changed, 44 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13f74b82d..f717be694 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1188,22 +1188,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "heed" -version = "0.10.6" -dependencies = [ - "byteorder", - "heed-traits 0.7.0", - "heed-types 0.7.2", - "libc", - "lmdb-rkv-sys", - "once_cell", - "page_size", - "synchronoise", - "url", - "zerocopy", -] - [[package]] name = "heed" version = "0.10.6" @@ -1211,8 +1195,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" dependencies = [ "byteorder", - "heed-traits 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heed-types 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "heed-traits", + "heed-types", "libc", "lmdb-rkv-sys", "once_cell", @@ -1223,27 +1207,12 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "heed-traits" -version = "0.7.0" - [[package]] name = "heed-traits" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" -[[package]] -name = "heed-types" -version = "0.7.2" -dependencies = [ - "bincode", - "heed-traits 0.7.0", - "serde", - "serde_json", - "zerocopy", -] - [[package]] name = "heed-types" version = "0.7.2" @@ -1251,7 +1220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" dependencies = [ "bincode", - "heed-traits 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "heed-traits", "serde", "serde_json", "zerocopy", @@ -1646,7 +1615,7 @@ dependencies = [ "futures", "futures-util", "grenad", - "heed 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "heed", "http", "indexmap", "jemallocator", @@ -1738,7 +1707,7 @@ dependencies = [ "fst", "fxhash", "grenad", - "heed 0.10.6", + "heed", "human_format", "itertools", "jemallocator", diff --git a/src/data/updates.rs b/src/data/updates.rs index d12f271c8..6bb9b20f2 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -7,7 +7,7 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use milli::update_store::UpdateStatus; use super::Data; -use crate::updates::UpdateMeta; +use crate::updates::{UpdateMeta, UpdateResult}; impl Data { pub async fn add_documents( @@ -47,7 +47,7 @@ impl Data { #[inline] - pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result>> { + pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result>> { self.update_queue.get_update_status(uid) } } diff --git a/src/updates/mod.rs b/src/updates/mod.rs index cd8052108..93cbcc008 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -13,7 +13,7 @@ use flate2::read::GzDecoder; use grenad::CompressionType; use log::info; use milli::Index; -use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod }; +use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod, DocumentAdditionResult }; use milli::update_store::{UpdateStore, UpdateHandler as Handler, UpdateStatus, Processing, Processed, Failed}; use rayon::ThreadPool; use serde::{Serialize, Deserialize}; @@ -41,13 +41,19 @@ pub enum UpdateMetaProgress { }, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum UpdateResult { + DocumentsAddition(DocumentAdditionResult), + Other, +} + #[derive(Clone)] pub struct UpdateQueue { - inner: Arc>, + inner: Arc>, } impl Deref for UpdateQueue { - type Target = Arc>; + type Target = Arc>; fn deref(&self) -> &Self::Target { &self.inner @@ -163,7 +169,7 @@ impl UpdateHandler { method: IndexDocumentsMethod, content: &[u8], update_builder: UpdateBuilder, - ) -> Result<()> { + ) -> Result { // We must use the write transaction of the update here. let mut wtxn = self.indexes.write_txn()?; let mut builder = update_builder.index_documents(&mut wtxn, &self.indexes); @@ -180,23 +186,29 @@ impl UpdateHandler { let result = builder.execute(reader, |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); match result { - Ok(()) => wtxn.commit().map_err(Into::into), + Ok(addition_result) => wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into), Err(e) => Err(e.into()) } } - fn clear_documents(&self, update_builder: UpdateBuilder) -> Result<()> { + fn clear_documents(&self, update_builder: UpdateBuilder) -> Result { // We must use the write transaction of the update here. let mut wtxn = self.indexes.write_txn()?; let builder = update_builder.clear_documents(&mut wtxn, &self.indexes); match builder.execute() { - Ok(_count) => wtxn.commit().map_err(Into::into), + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), Err(e) => Err(e.into()) } } - fn update_settings(&self, settings: &Settings, update_builder: UpdateBuilder) -> Result<()> { + fn update_settings(&self, settings: &Settings, update_builder: UpdateBuilder) -> Result { // We must use the write transaction of the update here. let mut wtxn = self.indexes.write_txn()?; let mut builder = update_builder.settings(&mut wtxn, &self.indexes); @@ -233,12 +245,15 @@ impl UpdateHandler { let result = builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); match result { - Ok(_count) => wtxn.commit().map_err(Into::into), + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), Err(e) => Err(e.into()) } } - fn update_facets(&self, levels: &Facets, update_builder: UpdateBuilder) -> Result<()> { + fn update_facets(&self, levels: &Facets, update_builder: UpdateBuilder) -> Result { // We must use the write transaction of the update here. let mut wtxn = self.indexes.write_txn()?; let mut builder = update_builder.facets(&mut wtxn, &self.indexes); @@ -249,38 +264,37 @@ impl UpdateHandler { builder.min_level_size(value); } match builder.execute() { - Ok(()) => wtxn.commit().map_err(Into::into), + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), Err(e) => Err(e.into()) } } } -impl Handler for UpdateHandler { +impl Handler for UpdateHandler { fn handle_update( &mut self, update_id: u64, meta: Processing, content: &[u8] - ) -> Result, Failed> { + ) -> Result, Failed> { use UpdateMeta::*; let update_builder = self.update_buidler(update_id); - let result: anyhow::Result<()> = match meta.meta() { + let result = match meta.meta() { DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), ClearDocuments => self.clear_documents(update_builder), Settings(settings) => self.update_settings(settings, update_builder), Facets(levels) => self.update_facets(levels, update_builder), }; - let new_meta = match result { - Ok(()) => format!("valid update content"), - Err(e) => format!("error while processing update content: {:?}", e), - }; - - let meta = meta.process(new_meta); - - Ok(meta) + match result { + Ok(result) => Ok(meta.process(result)), + Err(e) => Err(meta.fail(e.to_string())), + } } } @@ -302,7 +316,7 @@ impl UpdateQueue { } #[inline] - pub fn get_update_status(&self, update_id: u64) -> Result>> { + pub fn get_update_status(&self, update_id: u64) -> Result>> { Ok(self.inner.meta(update_id)?) } } From 12ee7b9b133a1c80a2321b2950ce481628885444 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 30 Dec 2020 19:17:13 +0100 Subject: [PATCH 011/522] impl get all updates --- src/data/updates.rs | 16 ++++++++++++++++ src/routes/index.rs | 16 +++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 6bb9b20f2..bb96c53c6 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -50,4 +50,20 @@ impl Data { pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result>> { self.update_queue.get_update_status(uid) } + + pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { + let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { + let mut metas = processing + .map(UpdateStatus::from) + .into_iter() + .chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + .chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + .chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + .chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + .collect::>(); + metas.sort_by(|a, b| a.id().cmp(&b.id())); + Ok(metas) + })?; + Ok(result) + } } diff --git a/src/routes/index.rs b/src/routes/index.rs index eb961a8dd..515e771e1 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -124,8 +124,18 @@ async fn get_update_status( #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] async fn get_all_updates_status( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() + let result = data.get_updates_status(&path.index_uid); + match result { + Ok(metas) => { + let json = serde_json::to_string(&metas).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + todo!() + } + } } From d1e9ded76fc8e30ad14863ce4581124faadc634b Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 31 Dec 2020 00:50:30 +0100 Subject: [PATCH 012/522] setting builder takes ownership --- Cargo.lock | 2 +- src/updates/mod.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f717be694..1f34a388e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1588,7 +1588,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.15.0" +version = "0.17.0" dependencies = [ "actix-http", ] diff --git a/src/updates/mod.rs b/src/updates/mod.rs index 93cbcc008..96ad3e16c 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -216,7 +216,7 @@ impl UpdateHandler { // We transpose the settings JSON struct into a real setting update. if let Some(ref names) = settings.searchable_attributes { match names { - Some(names) => builder.set_searchable_fields(&names), + Some(names) => builder.set_searchable_fields(names.clone()), None => builder.reset_searchable_fields(), } } @@ -224,20 +224,20 @@ impl UpdateHandler { // We transpose the settings JSON struct into a real setting update. if let Some(ref names) = settings.displayed_attributes { match names { - Some(names) => builder.set_displayed_fields(&names), + Some(names) => builder.set_displayed_fields(names.clone()), None => builder.reset_displayed_fields(), } } // We transpose the settings JSON struct into a real setting update. if let Some(ref facet_types) = settings.faceted_attributes { - builder.set_faceted_fields(&facet_types); + builder.set_faceted_fields(facet_types.clone()); } // We transpose the settings JSON struct into a real setting update. if let Some(ref criteria) = settings.criteria { match criteria { - Some(criteria) => builder.set_criteria(&criteria), + Some(criteria) => builder.set_criteria(criteria.clone()), None => builder.reset_criteria(), } } From b4d447b5cbf7ef94e462673185da54de8c183427 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 1 Jan 2021 16:59:49 +0100 Subject: [PATCH 013/522] temp --- Cargo.lock | 2 +- src/data/mod.rs | 36 ++++++- src/data/updates.rs | 14 ++- src/lib.rs | 2 +- src/routes/mod.rs | 2 +- src/routes/setting.rs | 226 ---------------------------------------- src/updates/mod.rs | 10 +- src/updates/settings.rs | 16 ++- 8 files changed, 70 insertions(+), 238 deletions(-) delete mode 100644 src/routes/setting.rs diff --git a/Cargo.lock b/Cargo.lock index 1f34a388e..f717be694 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1588,7 +1588,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.17.0" +version = "0.15.0" dependencies = [ "actix-http", ] diff --git a/src/data/mod.rs b/src/data/mod.rs index a699c9d20..9d64052af 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,6 +3,7 @@ mod updates; pub use search::{SearchQuery, SearchResult}; +use std::collections::HashMap; use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; @@ -10,7 +11,7 @@ use std::sync::Arc; use milli::Index; use sha2::Digest; -use crate::option::Opt; +use crate::{option::Opt, updates::Settings}; use crate::updates::UpdateQueue; #[derive(Clone)] @@ -82,6 +83,39 @@ impl Data { Ok(Data { inner }) } + pub fn settings>(&self, _index: S) -> anyhow::Result { + let txn = self.indexes.env.read_txn()?; + let fields_map = self.indexes.fields_ids_map(&txn)?; + println!("fields_map: {:?}", fields_map); + + let displayed_attributes = self.indexes + .displayed_fields(&txn)? + .map(|fields| {println!("{:?}", fields); fields.iter().filter_map(|f| fields_map.name(*f).map(String::from)).collect()}) + .unwrap_or_else(|| vec!["*".to_string()]); + + let searchable_attributes = self.indexes + .searchable_fields(&txn)? + .map(|fields| fields + .iter() + .filter_map(|f| fields_map.name(*f).map(String::from)) + .collect()) + .unwrap_or_else(|| vec!["*".to_string()]); + + let faceted_attributes = self.indexes + .faceted_fields(&txn)? + .iter() + .filter_map(|(f, t)| Some((fields_map.name(*f)?.to_string(), t.to_string()))) + .collect::>() + .into(); + + Ok(Settings { + displayed_attributes: Some(Some(displayed_attributes)), + searchable_attributes: Some(Some(searchable_attributes)), + faceted_attributes: Some(faceted_attributes), + criteria: None, + }) + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/src/data/updates.rs b/src/data/updates.rs index bb96c53c6..0654f6fd3 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -7,7 +7,7 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use milli::update_store::UpdateStatus; use super::Data; -use crate::updates::{UpdateMeta, UpdateResult}; +use crate::updates::{UpdateMeta, UpdateResult, UpdateStatusResponse, Settings}; impl Data { pub async fn add_documents( @@ -16,7 +16,7 @@ impl Data { method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, - ) -> anyhow::Result> + ) -> anyhow::Result where B: Deref, E: std::error::Error + Send + Sync + 'static, @@ -45,6 +45,16 @@ impl Data { Ok(update.into()) } + pub async fn update_settings>( + &self, + _index: S, + settings: Settings + ) -> anyhow::Result { + let meta = UpdateMeta::Settings(settings); + let queue = self.update_queue.clone(); + let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &[])).await??; + Ok(update.into()) + } #[inline] pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result>> { diff --git a/src/lib.rs b/src/lib.rs index 2c625588d..e0ae9aedb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,7 +46,7 @@ pub fn create_app( .configure(routes::document::services) .configure(routes::index::services) .configure(routes::search::services) - .configure(routes::setting::services) + .configure(routes::settings::services) .configure(routes::stop_words::services) .configure(routes::synonym::services) .configure(routes::health::services) diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 4b0813067..e1345fa08 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -6,7 +6,7 @@ pub mod health; pub mod index; pub mod key; pub mod search; -pub mod setting; +pub mod settings; pub mod stats; pub mod stop_words; pub mod synonym; diff --git a/src/routes/setting.rs b/src/routes/setting.rs deleted file mode 100644 index a405de45a..000000000 --- a/src/routes/setting.rs +++ /dev/null @@ -1,226 +0,0 @@ -use std::collections::BTreeSet; - -use actix_web::{delete, get, post}; -use actix_web::{web, HttpResponse}; - -use crate::Data; -use crate::error::ResponseError; -use crate::helpers::Authentication; -use crate::updates::Settings; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(update_all) - .service(get_all) - .service(delete_all) - .service(get_rules) - .service(update_rules) - .service(delete_rules) - .service(get_distinct) - .service(update_distinct) - .service(delete_distinct) - .service(get_searchable) - .service(update_searchable) - .service(delete_searchable) - .service(get_displayed) - .service(update_displayed) - .service(delete_displayed) - .service(get_attributes_for_faceting) - .service(delete_attributes_for_faceting) - .service(update_attributes_for_faceting); -} - - -#[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] -async fn update_all( - _data: web::Data, - _path: web::Path, - _body: web::Json, -) -> Result { - todo!() -} - -#[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] -async fn get_all( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] -async fn delete_all( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[get( - "/indexes/{index_uid}/settings/ranking-rules", - wrap = "Authentication::Private" -)] -async fn get_rules( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[post( - "/indexes/{index_uid}/settings/ranking-rules", - wrap = "Authentication::Private" -)] -async fn update_rules( - _data: web::Data, - _path: web::Path, - _body: web::Json>>, -) -> Result { - todo!() -} - -#[delete( - "/indexes/{index_uid}/settings/ranking-rules", - wrap = "Authentication::Private" -)] -async fn delete_rules( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[get( - "/indexes/{index_uid}/settings/distinct-attribute", - wrap = "Authentication::Private" -)] -async fn get_distinct( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[post( - "/indexes/{index_uid}/settings/distinct-attribute", - wrap = "Authentication::Private" -)] -async fn update_distinct( - _data: web::Data, - _path: web::Path, - _body: web::Json>, -) -> Result { - todo!() -} - -#[delete( - "/indexes/{index_uid}/settings/distinct-attribute", - wrap = "Authentication::Private" -)] -async fn delete_distinct( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[get( - "/indexes/{index_uid}/settings/searchable-attributes", - wrap = "Authentication::Private" -)] -async fn get_searchable( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[post( - "/indexes/{index_uid}/settings/searchable-attributes", - wrap = "Authentication::Private" -)] -async fn update_searchable( - _data: web::Data, - _path: web::Path, - _body: web::Json>>, -) -> Result { - todo!() -} - -#[delete( - "/indexes/{index_uid}/settings/searchable-attributes", - wrap = "Authentication::Private" -)] -async fn delete_searchable( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[get( - "/indexes/{index_uid}/settings/displayed-attributes", - wrap = "Authentication::Private" -)] -async fn get_displayed( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[post( - "/indexes/{index_uid}/settings/displayed-attributes", - wrap = "Authentication::Private" -)] -async fn update_displayed( - _data: web::Data, - _path: web::Path, - _body: web::Json>>, -) -> Result { - todo!() -} - -#[delete( - "/indexes/{index_uid}/settings/displayed-attributes", - wrap = "Authentication::Private" -)] -async fn delete_displayed( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[get( - "/indexes/{index_uid}/settings/attributes-for-faceting", - wrap = "Authentication::Private" -)] -async fn get_attributes_for_faceting( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[post( - "/indexes/{index_uid}/settings/attributes-for-faceting", - wrap = "Authentication::Private" -)] -async fn update_attributes_for_faceting( - _data: web::Data, - _path: web::Path, - _body: web::Json>>, -) -> Result { - todo!() -} - -#[delete( - "/indexes/{index_uid}/settings/attributes-for-faceting", - wrap = "Authentication::Private" -)] -async fn delete_attributes_for_faceting( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} diff --git a/src/updates/mod.rs b/src/updates/mod.rs index 96ad3e16c..d92a3b16f 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -6,6 +6,7 @@ use std::io; use std::sync::Arc; use std::ops::Deref; use std::fs::create_dir_all; +use std::collections::HashMap; use anyhow::Result; use byte_unit::Byte; @@ -21,6 +22,8 @@ use structopt::StructOpt; use crate::option::Opt; +pub type UpdateStatusResponse = UpdateStatus; + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] pub enum UpdateMeta { @@ -231,7 +234,8 @@ impl UpdateHandler { // We transpose the settings JSON struct into a real setting update. if let Some(ref facet_types) = settings.faceted_attributes { - builder.set_faceted_fields(facet_types.clone()); + let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + builder.set_faceted_fields(facet_types); } // We transpose the settings JSON struct into a real setting update. @@ -245,7 +249,7 @@ impl UpdateHandler { let result = builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); match result { - Ok(_count) => wtxn + Ok(()) => wtxn .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), @@ -316,7 +320,7 @@ impl UpdateQueue { } #[inline] - pub fn get_update_status(&self, update_id: u64) -> Result>> { + pub fn get_update_status(&self, update_id: u64) -> Result> { Ok(self.inner.meta(update_id)?) } } diff --git a/src/updates/settings.rs b/src/updates/settings.rs index 58414152a..91381edc5 100644 --- a/src/updates/settings.rs +++ b/src/updates/settings.rs @@ -11,7 +11,7 @@ where T: Deserialize<'de>, Deserialize::deserialize(deserializer).map(Some) } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct Settings { @@ -30,7 +30,7 @@ pub struct Settings { pub searchable_attributes: Option>>, #[serde(default)] - pub faceted_attributes: Option>, + pub faceted_attributes: Option>>, #[serde( default, @@ -40,6 +40,17 @@ pub struct Settings { pub criteria: Option>>, } +impl Settings { + pub fn cleared() -> Self { + Self { + displayed_attributes: Some(None), + searchable_attributes: Some(None), + faceted_attributes: Some(None), + criteria: Some(None), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -48,4 +59,3 @@ pub struct Facets { pub level_group_size: Option, pub min_level_size: Option, } - From b07e21ab3c58a5870488b6d29b6ddacac64e53c2 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 5 Jan 2021 00:21:42 +0100 Subject: [PATCH 014/522] temp --- .../settings/attributes_for_faceting.rs | 43 ++++ src/routes/settings/displayed_attributes.rs | 25 +++ src/routes/settings/distinct_attributes.rs | 36 ++++ src/routes/settings/mod.rs | 190 ++++++++++++++++++ src/routes/settings/ranking_rules.rs | 23 +++ src/routes/settings/searchable_attributes.rs | 34 ++++ src/routes/settings/stop_words.rs | 33 +++ src/routes/settings/synonyms.rs | 43 ++++ 8 files changed, 427 insertions(+) create mode 100644 src/routes/settings/attributes_for_faceting.rs create mode 100644 src/routes/settings/displayed_attributes.rs create mode 100644 src/routes/settings/distinct_attributes.rs create mode 100644 src/routes/settings/mod.rs create mode 100644 src/routes/settings/ranking_rules.rs create mode 100644 src/routes/settings/searchable_attributes.rs create mode 100644 src/routes/settings/stop_words.rs create mode 100644 src/routes/settings/synonyms.rs diff --git a/src/routes/settings/attributes_for_faceting.rs b/src/routes/settings/attributes_for_faceting.rs new file mode 100644 index 000000000..6c881cff3 --- /dev/null +++ b/src/routes/settings/attributes_for_faceting.rs @@ -0,0 +1,43 @@ +use actix_web::{web, HttpResponse, get}; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::make_update_delete_routes; +use crate::Data; + +#[get( + "/indexes/{index_uid}/settings/attributes-for-faceting", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + index_uid: web::Path, +) -> Result { + let index = data + .db + .load() + .open_index(&index_uid.as_ref()) + .ok_or(Error::index_not_found(&index_uid.as_ref()))?; + + let attributes_for_faceting = data.db.load().main_read::<_, _, ResponseError>(|reader| { + let schema = index.main.schema(reader)?; + let attrs = index.main.attributes_for_faceting(reader)?; + let attr_names = match (&schema, &attrs) { + (Some(schema), Some(attrs)) => attrs + .iter() + .filter_map(|&id| schema.name(id)) + .map(str::to_string) + .collect(), + _ => vec![], + }; + Ok(attr_names) + })?; + + Ok(HttpResponse::Ok().json(attributes_for_faceting)) +} + +make_update_delete_routes!( + "/indexes/{index_uid}/settings/attributes-for-faceting", + Vec, + attributes_for_faceting +); diff --git a/src/routes/settings/displayed_attributes.rs b/src/routes/settings/displayed_attributes.rs new file mode 100644 index 000000000..b9f36f718 --- /dev/null +++ b/src/routes/settings/displayed_attributes.rs @@ -0,0 +1,25 @@ +use std::collections::HashSet; + +use actix_web::{web, HttpResponse, get}; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::make_update_delete_routes; +use crate::Data; + +#[get( + "/indexes/{index_uid}/settings/displayed-attributes", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + index_uid: web::Path, +) -> Result { + todo!() +} + +make_update_delete_routes!( + "/indexes/{index_uid}/settings/displayed-attributes", + HashSet, + displayed_attributes +); diff --git a/src/routes/settings/distinct_attributes.rs b/src/routes/settings/distinct_attributes.rs new file mode 100644 index 000000000..7b991f861 --- /dev/null +++ b/src/routes/settings/distinct_attributes.rs @@ -0,0 +1,36 @@ +use crate::make_update_delete_routes; +use actix_web::{web, HttpResponse, get}; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::Data; + +#[get( + "/indexes/{index_uid}/settings/distinct-attribute", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + index_uid: web::Path, +) -> Result { + let index = data + .db + .load() + .open_index(&index_uid.as_ref()) + .ok_or(Error::index_not_found(&index_uid.as_ref()))?; + let reader = data.db.load().main_read_txn()?; + let distinct_attribute_id = index.main.distinct_attribute(&reader)?; + let schema = index.main.schema(&reader)?; + let distinct_attribute = match (schema, distinct_attribute_id) { + (Some(schema), Some(id)) => schema.name(id).map(str::to_string), + _ => None, + }; + + Ok(HttpResponse::Ok().json(distinct_attribute)) +} + +make_update_delete_routes!( + "/indexes/{index_uid}/settings/distinct-attribute", + String, + distinct_attribute +); diff --git a/src/routes/settings/mod.rs b/src/routes/settings/mod.rs new file mode 100644 index 000000000..d54ad469c --- /dev/null +++ b/src/routes/settings/mod.rs @@ -0,0 +1,190 @@ +use actix_web::{web, HttpResponse, delete, get, post}; +use log::error; + +use crate::Data; +use crate::error::ResponseError; +use crate::updates::Settings; +use crate::helpers::Authentication; + +#[macro_export] +macro_rules! make_setting_route { + ($route:literal, $type:ty, $attr:ident) => { + mod $attr { + use actix_web::{web, HttpResponse}; + + use crate::data; + use crate::error::ResponseError; + use crate::helpers::Authentication; + use crate::updates::Settings; + + #[actix_web::delete($route, wrap = "Authentication::Private")] + pub async fn delete( + data: web::Data, + index_uid: web::Path, + ) -> Result { + use crate::updates::Settings; + let settings = Settings { + $attr: Some(None), + ..Default::default() + }; + match data.update_settings(index_uid.as_ref(), settings).await { + Ok(update_status) => { + let json = serde_json::to_string(&update_status).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + log::error!("{}", e); + unimplemented!(); + } + } + } + + #[actix_web::post($route, wrap = "Authentication::Private")] + pub async fn update( + data: actix_web::web::Data, + index_uid: actix_web::web::Path, + body: actix_web::web::Json>, + ) -> std::result::Result { + let settings = Settings { + $attr: Some(body.into_inner()), + ..Default::default() + }; + + match data.update_settings(index_uid.as_ref(), settings).await { + Ok(update_status) => { + let json = serde_json::to_string(&update_status).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + log::error!("{}", e); + unimplemented!(); + } + } + } + + #[actix_web::get($route, wrap = "Authentication::Private")] + pub async fn get( + data: actix_web::web::Data, + index_uid: actix_web::web::Path, + ) -> std::result::Result { + match data.settings(index_uid.as_ref()) { + Ok(settings) => { + let setting = settings.$attr; + let json = serde_json::to_string(&setting).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + log::error!("{}", e); + unimplemented!(); + } + } + } + } + }; +} + +make_setting_route!( + "/indexes/{index_uid}/settings/attributes-for-faceting", + std::collections::HashMap, + faceted_attributes +); + +make_setting_route!( + "/indexes/{index_uid}/settings/displayed-attributes", + Vec, + displayed_attributes +); + +make_setting_route!( + "/indexes/{index_uid}/settings/searchable-attributes", + Vec, + searchable_attributes +); + +//make_setting_route!( + //"/indexes/{index_uid}/settings/distinct-attribute", + //String, + //distinct_attribute +//); + +//make_setting_route!( + //"/indexes/{index_uid}/settings/ranking-rules", + //Vec, + //ranking_rules +//); + +macro_rules! create_services { + ($($mod:ident),*) => { + pub fn services(cfg: &mut web::ServiceConfig) { + cfg + .service(update_all) + .service(get_all) + .service(delete_all) + $( + .service($mod::get) + .service($mod::update) + .service($mod::delete) + )*; + } + }; +} + +create_services!( + faceted_attributes, + displayed_attributes, + searchable_attributes +); + +#[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] +async fn update_all( + data: web::Data, + index_uid: web::Path, + body: web::Json, +) -> Result { + match data.update_settings(index_uid.as_ref(), body.into_inner()).await { + Ok(update_result) => { + let json = serde_json::to_string(&update_result).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + unimplemented!(); + } + } +} + +#[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] +async fn get_all( + data: web::Data, + index_uid: web::Path, +) -> Result { + match data.settings(index_uid.as_ref()) { + Ok(settings) => { + let json = serde_json::to_string(&settings).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + unimplemented!(); + } + } +} + +#[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] +async fn delete_all( + data: web::Data, + index_uid: web::Path, +) -> Result { + let settings = Settings::cleared(); + match data.update_settings(index_uid.as_ref(), settings).await { + Ok(update_result) => { + let json = serde_json::to_string(&update_result).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + unimplemented!(); + } + } +} + diff --git a/src/routes/settings/ranking_rules.rs b/src/routes/settings/ranking_rules.rs new file mode 100644 index 000000000..e0872954a --- /dev/null +++ b/src/routes/settings/ranking_rules.rs @@ -0,0 +1,23 @@ +use crate::make_update_delete_routes; +use actix_web::{web, HttpResponse, get}; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::Data; + +#[get( + "/indexes/{index_uid}/settings/ranking-rules", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + index_uid: web::Path, +) -> Result { + todo!() +} + +make_update_delete_routes!( + "/indexes/{index_uid}/settings/ranking-rules", + Vec, + ranking_rules +); diff --git a/src/routes/settings/searchable_attributes.rs b/src/routes/settings/searchable_attributes.rs new file mode 100644 index 000000000..a337b0435 --- /dev/null +++ b/src/routes/settings/searchable_attributes.rs @@ -0,0 +1,34 @@ +use actix_web::{web, HttpResponse, get}; + +use crate::data::get_indexed_attributes; +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::make_update_delete_routes; +use crate::Data; + +#[get( + "/indexes/{index_uid}/settings/searchable-attributes", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + index_uid: web::Path, +) -> Result { + let index = data + .db + .load() + .open_index(&index_uid.as_ref()) + + .ok_or(Error::index_not_found(&index_uid.as_ref()))?; + let reader = data.db.load().main_read_txn()?; + let schema = index.main.schema(&reader)?; + let searchable_attributes: Option> = schema.as_ref().map(get_indexed_attributes); + + Ok(HttpResponse::Ok().json(searchable_attributes)) +} + +make_update_delete_routes!( + "/indexes/{index_uid}/settings/searchable-attributes", + Vec, + searchable_attributes +); diff --git a/src/routes/settings/stop_words.rs b/src/routes/settings/stop_words.rs new file mode 100644 index 000000000..05a753f46 --- /dev/null +++ b/src/routes/settings/stop_words.rs @@ -0,0 +1,33 @@ +use std::collections::BTreeSet; + +use crate::make_update_delete_routes; +use actix_web::{web, HttpResponse, get}; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::Data; + +#[get( + "/indexes/{index_uid}/settings/stop-words", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + index_uid: web::Path, +) -> Result { + let index = data + .db + .load() + .open_index(&index_uid.as_ref()) + .ok_or(Error::index_not_found(&index_uid.as_ref()))?; + let reader = data.db.load().main_read_txn()?; + let stop_words = index.main.stop_words(&reader)?; + + Ok(HttpResponse::Ok().json(stop_words)) +} + +make_update_delete_routes!( + "/indexes/{index_uid}/settings/stop-words", + BTreeSet, + stop_words +); diff --git a/src/routes/settings/synonyms.rs b/src/routes/settings/synonyms.rs new file mode 100644 index 000000000..e5b5b2afd --- /dev/null +++ b/src/routes/settings/synonyms.rs @@ -0,0 +1,43 @@ +use std::collections::BTreeMap; + +use actix_web::{web, HttpResponse, get}; +use indexmap::IndexMap; + +use crate::error::{Error, ResponseError}; +use crate::helpers::Authentication; +use crate::make_update_delete_routes; +use crate::Data; + +#[get( + "/indexes/{index_uid}/settings/synonyms", + wrap = "Authentication::Private" +)] +async fn get( + data: web::Data, + index_uid: web::Path, +) -> Result { + let index = data + .db + .load() + .open_index(&index_uid.as_ref()) + .ok_or(Error::index_not_found(&index_uid.as_ref()))?; + + let reader = data.db.load().main_read_txn()?; + + let synonyms_list = index.main.synonyms(&reader)?; + + let mut synonyms = IndexMap::new(); + let index_synonyms = &index.synonyms; + for synonym in synonyms_list { + let list = index_synonyms.synonyms(&reader, synonym.as_bytes())?; + synonyms.insert(synonym, list); + } + + Ok(HttpResponse::Ok().json(synonyms)) +} + +make_update_delete_routes!( + "/indexes/{index_uid}/settings/synonyms", + BTreeMap>, + synonyms +); From ddd778971351d8eb5b937813ea5b375b1af9b68c Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 13 Jan 2021 17:50:36 +0100 Subject: [PATCH 015/522] WIP: IndexController --- Cargo.lock | 28 +++--- Cargo.toml | 4 +- src/data/mod.rs | 26 +++-- src/data/search.rs | 36 +++---- src/index_controller/mod.rs | 187 ++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 +- 6 files changed, 230 insertions(+), 55 deletions(-) create mode 100644 src/index_controller/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f717be694..4415db555 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -777,6 +777,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + [[package]] name = "debugid" version = "0.7.2" @@ -1201,7 +1211,6 @@ dependencies = [ "lmdb-rkv-sys", "once_cell", "page_size", - "serde", "synchronoise", "url", "zerocopy", @@ -1588,7 +1597,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.15.0" +version = "0.18.0" dependencies = [ "actix-http", ] @@ -1609,6 +1618,7 @@ dependencies = [ "bytes 0.6.0", "chrono", "crossbeam-channel", + "dashmap", "env_logger 0.8.2", "flate2", "fst", @@ -1627,6 +1637,7 @@ dependencies = [ "milli", "mime", "once_cell", + "page_size", "rand 0.7.3", "rayon", "regex", @@ -1699,7 +1710,6 @@ dependencies = [ "bstr", "byte-unit", "byteorder", - "chrono", "crossbeam-channel", "csv", "either", @@ -1714,13 +1724,13 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", - "meilisearch-tokenizer", "memmap", "near-proximity", "num-traits", "obkv", "once_cell", "ordered-float", + "page_size", "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", "pest_derive", "rayon", @@ -1729,7 +1739,6 @@ dependencies = [ "roaring", "serde", "serde_json", - "serde_millis", "slice-group-by", "smallstr", "smallvec", @@ -2596,15 +2605,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_millis" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e2dc780ca5ee2c369d1d01d100270203c4ff923d2a4264812d723766434d00" -dependencies = [ - "serde", -] - [[package]] name = "serde_qs" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index fb8531a96..47bbc09df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = "0.10.6" +heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] } http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" @@ -58,6 +58,8 @@ tokio = { version = "0.2", features = ["full"] } ureq = { version = "1.5.1", default-features = false, features = ["tls"] } walkdir = "2.3.1" whoami = "1.0.0" +dashmap = "4.0.2" +page_size = "0.4.2" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 9d64052af..944690e0c 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,7 +3,6 @@ mod updates; pub use search::{SearchQuery, SearchResult}; -use std::collections::HashMap; use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; @@ -13,6 +12,7 @@ use sha2::Digest; use crate::{option::Opt, updates::Settings}; use crate::updates::UpdateQueue; +use crate::index_controller::IndexController; #[derive(Clone)] pub struct Data { @@ -29,7 +29,7 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { - pub indexes: Arc, + pub indexes: Arc, pub update_queue: Arc, api_keys: ApiKeys, options: Opt, @@ -62,9 +62,7 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { let db_size = options.max_mdb_size.get_bytes() as usize; - let path = options.db_path.join("main"); - create_dir_all(&path)?; - let indexes = Index::new(&path, Some(db_size))?; + let indexes = IndexController::new(&options.db_path)?; let indexes = Arc::new(indexes); let update_queue = Arc::new(UpdateQueue::new(&options, indexes.clone())?); @@ -90,28 +88,26 @@ impl Data { let displayed_attributes = self.indexes .displayed_fields(&txn)? - .map(|fields| {println!("{:?}", fields); fields.iter().filter_map(|f| fields_map.name(*f).map(String::from)).collect()}) + .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); let searchable_attributes = self.indexes .searchable_fields(&txn)? .map(|fields| fields - .iter() - .filter_map(|f| fields_map.name(*f).map(String::from)) + .into_iter() + .map(String::from) .collect()) .unwrap_or_else(|| vec!["*".to_string()]); - let faceted_attributes = self.indexes - .faceted_fields(&txn)? - .iter() - .filter_map(|(f, t)| Some((fields_map.name(*f)?.to_string(), t.to_string()))) - .collect::>() - .into(); + let faceted_attributes = self.indexes.faceted_fields(&txn)? + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(); Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), searchable_attributes: Some(Some(searchable_attributes)), - faceted_attributes: Some(faceted_attributes), + faceted_attributes: Some(Some(faceted_attributes)), criteria: None, }) } diff --git a/src/data/search.rs b/src/data/search.rs index bd22a959b..44e0a54e6 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::collections::HashSet; use std::mem; use std::time::Instant; @@ -8,17 +7,22 @@ use serde::{Deserialize, Serialize}; use milli::{SearchResult as Results, obkv_to_json}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use crate::error::Error; + use super::Data; const DEFAULT_SEARCH_LIMIT: usize = 20; +const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } + #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] #[allow(dead_code)] pub struct SearchQuery { q: Option, offset: Option, - limit: Option, + #[serde(default = "default_search_limit")] + limit: usize, attributes_to_retrieve: Option>, attributes_to_crop: Option>, crop_length: Option, @@ -100,30 +104,18 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } impl Data { - pub fn search>(&self, _index: S, search_query: SearchQuery) -> anyhow::Result { + pub fn search>(&self, index: S, search_query: SearchQuery) -> anyhow::Result { let start = Instant::now(); - let index = &self.indexes; - let rtxn = index.read_txn()?; - - let mut search = index.search(&rtxn); - if let Some(query) = &search_query.q { - search.query(query); - } - - if let Some(offset) = search_query.offset { - search.offset(offset); - } - - let limit = search_query.limit.unwrap_or(DEFAULT_SEARCH_LIMIT); - search.limit(limit); - - let Results { found_words, documents_ids, nb_hits, .. } = search.execute().unwrap(); + let index = self.indexes + .get(index)? + .ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?; + let Results { found_words, documents_ids, nb_hits, .. } = index.search(search_query)?; let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); - let displayed_fields = match index.displayed_fields(&rtxn).unwrap() { - Some(fields) => Cow::Borrowed(fields), - None => Cow::Owned(fields_ids_map.iter().map(|(id, _)| id).collect()), + let displayed_fields = match index.displayed_fields_ids(&rtxn).unwrap() { + Some(fields) => fields, + None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; let attributes_to_highlight = match search_query.attributes_to_highlight { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs new file mode 100644 index 000000000..9dfa23ce5 --- /dev/null +++ b/src/index_controller/mod.rs @@ -0,0 +1,187 @@ +use std::fs::File; +use std::io::{Read, Write}; +use std::path::Path; +use std::ops::Deref; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use heed::types::{Str, SerdeBincode}; +use heed::{EnvOpenOptions, Env, Database}; +use milli::Index; +use serde::{Serialize, Deserialize}; + +use crate::data::{SearchQuery, SearchResult}; + +const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; +const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; +const INDEXES_DB_NAME: &str = "indexes_db"; + +trait UpdateStore {} + +pub struct IndexController { + update_store: U, + env: Env, + indexes_db: Database>, + indexes: DashMap, +} + +#[derive(Debug, Serialize, Deserialize)] +struct IndexControllerMeta { + open_options: EnvOpenOptions, + created_at: DateTime, +} + +impl IndexControllerMeta { + fn from_path(path: impl AsRef) -> Result> { + let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + if path.exists() { + let mut file = File::open(path)?; + let mut buffer = Vec::new(); + let n = file.read_to_end(&mut buffer)?; + let meta: IndexControllerMeta = serde_json::from_slice(&buffer[..n])?; + Ok(Some(meta)) + } else { + Ok(None) + } + } + + fn to_path(self, path: impl AsRef) -> Result<()> { + let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + if path.exists() { + Err(anyhow::anyhow!("Index controller metadata already exists")) + } else { + let mut file = File::create(path)?; + let json = serde_json::to_vec(&self)?; + file.write_all(&json)?; + Ok(()) + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct IndexMetadata { + created_at: DateTime, + open_options: EnvOpenOptions, + id: String, +} + +impl IndexMetadata { + fn open_index(&self) -> Result { + todo!() + } +} + +struct IndexView<'a, U> { + txn: heed::RoTxn<'a>, + index: &'a Index, + update_store: &'a U, +} + +struct IndexViewMut<'a, U> { + txn: heed::RwTxn<'a>, + index: &'a Index, + update_store: &'a U, +} + +impl<'a, U> Deref for IndexViewMut<'a, U> { + type Target = IndexView<'a, U>; + + fn deref(&self) -> &Self::Target { + IndexView { + txn: *self.txn, + index: self.index, + update_store: self.update_store, + } + } +} + +impl<'a, U: UpdateStore> IndexView<'a, U> { + fn search(&self, search_query: SearchQuery) -> Result { + let mut search = self.index.search(self.txn); + if let Some(query) = &search_query.q { + search.query(query); + } + + if let Some(offset) = search_query.offset { + search.offset(offset); + } + + let limit = search_query.limit; + search.limit(limit); + + Ok(search.execute()?) + } +} + +impl IndexController { + /// Open the index controller from meta found at path, and create a new one if no meta is + /// found. + pub fn new(path: impl AsRef, update_store: U) -> Result { + // If index controller metadata is present, we return the env, otherwise, we create a new + // metadata from scratch before returning a new env. + let env = match IndexControllerMeta::from_path(path)? { + Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, + None => { + let open_options = EnvOpenOptions::new() + .map_size(page_size::get() * 1000); + let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; + let created_at = Utc::now(); + let meta = IndexControllerMeta { open_options, created_at }; + meta.to_path(path)?; + env + } + }; + let indexes = DashMap::new(); + let indexes_db = match env.open_database(INDEXES_DB_NAME)? { + Some(indexes_db) => indexes_db, + None => env.create_database(INDEXES_DB_NAME)?, + }; + + Ok(Self { env, indexes, indexes_db, update_store }) + } + + pub fn get_or_create>(&mut self, name: S) -> Result> { + todo!() + } + + /// Get an index with read access to the db. The index are lazily loaded, meaning that we first + /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check + /// for metadata to launch the index. + pub fn get>(&self, name: S) -> Result>> { + match self.indexes.get(name.as_ref()) { + Some(index) => { + let txn = index.read_txn()?; + let update_store = &self.update_store; + Ok(Some(IndexView { index, update_store, txn })) + } + None => { + let txn = self.env.read_txn()?; + match self.indexes_db.get(&txn, name.as_ref())? { + Some(meta) => { + let index = meta.open_index()?; + self.indexes.insert(name.as_ref().to_owned(), index); + Ok(self.indexes.get(name.as_ref())) + } + None => Ok(None) + } + } + } + } + + pub fn get_mut>(&self, name: S) -> Result>> { + todo!() + } + + pub async fn delete_index>(&self, name:S) -> Result<()> { + todo!() + } + + pub async fn list_indices(&self) -> Result> { + todo!() + } + + pub async fn rename_index(&self, old: &str, new: &str) -> Result<()> { + todo!() + } +} diff --git a/src/lib.rs b/src/lib.rs index e0ae9aedb..f5dd79b0b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,9 +6,7 @@ pub mod helpers; pub mod option; pub mod routes; mod updates; -//pub mod analytics; -//pub mod snapshot; -//pub mod dump; +mod index_controller; use actix_http::Error; use actix_service::ServiceFactory; From d22fab5baec0fdc6029056e9c8af87acb1fddeae Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 13 Jan 2021 18:18:52 +0100 Subject: [PATCH 016/522] implement open index --- Cargo.lock | 13 ++++++++++++- src/index_controller/mod.rs | 9 +++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4415db555..40528367e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1710,6 +1710,7 @@ dependencies = [ "bstr", "byte-unit", "byteorder", + "chrono", "crossbeam-channel", "csv", "either", @@ -1724,13 +1725,13 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", + "meilisearch-tokenizer", "memmap", "near-proximity", "num-traits", "obkv", "once_cell", "ordered-float", - "page_size", "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", "pest_derive", "rayon", @@ -1739,6 +1740,7 @@ dependencies = [ "roaring", "serde", "serde_json", + "serde_millis", "slice-group-by", "smallstr", "smallvec", @@ -2605,6 +2607,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_millis" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e2dc780ca5ee2c369d1d01d100270203c4ff923d2a4264812d723766434d00" +dependencies = [ + "serde", +] + [[package]] name = "serde_qs" version = "0.8.2" diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 9dfa23ce5..0c6a95c1f 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -67,8 +67,9 @@ struct IndexMetadata { } impl IndexMetadata { - fn open_index(&self) -> Result { - todo!() + fn open_index(&self, path: impl AsRef) -> Result { + let path = path.as_ref().to_path_buf().push("indexes").push(&self.id); + Ok(Index::new(self.options, path)?) } } @@ -79,7 +80,7 @@ struct IndexView<'a, U> { } struct IndexViewMut<'a, U> { - txn: heed::RwTxn<'a>, + txn: heed::RwTxn<'a, 'a>, index: &'a Index, update_store: &'a U, } @@ -97,7 +98,7 @@ impl<'a, U> Deref for IndexViewMut<'a, U> { } impl<'a, U: UpdateStore> IndexView<'a, U> { - fn search(&self, search_query: SearchQuery) -> Result { + pub fn search(&self, search_query: SearchQuery) -> Result { let mut search = self.index.search(self.txn); if let Some(query) = &search_query.q { search.query(query); From 334933b874d811599f2e893ca561c81e32729b0a Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 13 Jan 2021 18:29:17 +0100 Subject: [PATCH 017/522] fix search --- src/data/search.rs | 9 +++++---- src/index_controller/mod.rs | 14 +++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index 44e0a54e6..8c8e0da69 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -109,11 +109,12 @@ impl Data { let index = self.indexes .get(index)? .ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?; - let Results { found_words, documents_ids, nb_hits, .. } = index.search(search_query)?; - let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(search_query)?; - let displayed_fields = match index.displayed_fields_ids(&rtxn).unwrap() { + let fields_ids_map = index.fields_ids_map()?; + + let displayed_fields = match index.displayed_fields_ids()? { Some(fields) => fields, None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; @@ -126,7 +127,7 @@ impl Data { let stop_words = fst::Set::default(); let highlighter = Highlighter::new(&stop_words); let mut documents = Vec::new(); - for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { + for (_id, obkv) in index.documents(&documents_ids)? { let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); highlighter.highlight_record(&mut object, &found_words, &attributes_to_highlight); documents.push(object); diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0c6a95c1f..25e5d4787 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -8,7 +8,7 @@ use chrono::{DateTime, Utc}; use dashmap::DashMap; use heed::types::{Str, SerdeBincode}; use heed::{EnvOpenOptions, Env, Database}; -use milli::Index; +use milli::{Index, FieldsIdsMap}; use serde::{Serialize, Deserialize}; use crate::data::{SearchQuery, SearchResult}; @@ -113,6 +113,18 @@ impl<'a, U: UpdateStore> IndexView<'a, U> { Ok(search.execute()?) } + + pub fn fields_ids_map(&self) -> Result { + self.index.fields_ids_map(self.txn) + } + + pub fn fields_displayed_fields_ids(&self) -> Result { + self.index.fields_displayed_fields_ids(self.txn) + } + + pub fn documents(&self, ids: &[u32]) -> Result> { + self.index.documents(self.txn, ids) + } } impl IndexController { From 686f9871806505151d9c624723a76c0570209187 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 14 Jan 2021 11:27:07 +0100 Subject: [PATCH 018/522] fix compile errors --- Cargo.lock | 2 + Cargo.toml | 3 +- src/data/mod.rs | 5 +-- src/data/search.rs | 22 +++++----- src/index_controller/mod.rs | 86 ++++++++++++++++++------------------- src/updates/mod.rs | 2 + 6 files changed, 59 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40528367e..e0457b4d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1211,6 +1211,7 @@ dependencies = [ "lmdb-rkv-sys", "once_cell", "page_size", + "serde", "synchronoise", "url", "zerocopy", @@ -1636,6 +1637,7 @@ dependencies = [ "memmap", "milli", "mime", + "obkv", "once_cell", "page_size", "rand 0.7.3", diff --git a/Cargo.toml b/Cargo.toml index 47bbc09df..c06847253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] } +heed = { version = "0.10.6", default-features = false, features = ["serde", "lmdb", "sync-read-txn"] } http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" @@ -60,6 +60,7 @@ walkdir = "2.3.1" whoami = "1.0.0" dashmap = "4.0.2" page_size = "0.4.2" +obkv = "0.1.1" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 944690e0c..4600531bf 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,11 +3,9 @@ mod updates; pub use search::{SearchQuery, SearchResult}; -use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; -use milli::Index; use sha2::Digest; use crate::{option::Opt, updates::Settings}; @@ -29,8 +27,7 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { - pub indexes: Arc, - pub update_queue: Arc, + pub indexes: Arc>, api_keys: ApiKeys, options: Opt, } diff --git a/src/data/search.rs b/src/data/search.rs index 8c8e0da69..69029d8a9 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -19,18 +19,18 @@ const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } #[serde(rename_all = "camelCase", deny_unknown_fields)] #[allow(dead_code)] pub struct SearchQuery { - q: Option, - offset: Option, + pub q: Option, + pub offset: Option, #[serde(default = "default_search_limit")] - limit: usize, - attributes_to_retrieve: Option>, - attributes_to_crop: Option>, - crop_length: Option, - attributes_to_highlight: Option>, - filters: Option, - matches: Option, - facet_filters: Option, - facets_distribution: Option>, + pub limit: usize, + pub attributes_to_retrieve: Option>, + pub attributes_to_crop: Option>, + pub crop_length: Option, + pub attributes_to_highlight: Option>, + pub filters: Option, + pub matches: Option, + pub facet_filters: Option, + pub facets_distribution: Option>, } #[derive(Serialize)] diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 25e5d4787..a3b29879a 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,25 +1,26 @@ use std::fs::File; use std::io::{Read, Write}; -use std::path::Path; -use std::ops::Deref; +use std::path::{Path, PathBuf}; use anyhow::Result; use chrono::{DateTime, Utc}; use dashmap::DashMap; +use dashmap::mapref::one::Ref; use heed::types::{Str, SerdeBincode}; use heed::{EnvOpenOptions, Env, Database}; -use milli::{Index, FieldsIdsMap}; +use milli::{Index, FieldsIdsMap, SearchResult, FieldId}; use serde::{Serialize, Deserialize}; -use crate::data::{SearchQuery, SearchResult}; +use crate::data::SearchQuery; const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; const INDEXES_DB_NAME: &str = "indexes_db"; -trait UpdateStore {} +pub trait UpdateStore {} pub struct IndexController { + path: PathBuf, update_store: U, env: Env, indexes_db: Database>, @@ -34,7 +35,8 @@ struct IndexControllerMeta { impl IndexControllerMeta { fn from_path(path: impl AsRef) -> Result> { - let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); if path.exists() { let mut file = File::open(path)?; let mut buffer = Vec::new(); @@ -47,7 +49,8 @@ impl IndexControllerMeta { } fn to_path(self, path: impl AsRef) -> Result<()> { - let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); if path.exists() { Err(anyhow::anyhow!("Index controller metadata already exists")) } else { @@ -67,39 +70,24 @@ struct IndexMetadata { } impl IndexMetadata { - fn open_index(&self, path: impl AsRef) -> Result { - let path = path.as_ref().to_path_buf().push("indexes").push(&self.id); - Ok(Index::new(self.options, path)?) + fn open_index(&self, path: impl AsRef) -> Result { + // create a path in the form "db_path/indexes/index_id" + let mut path = path.as_ref().to_path_buf(); + path.push("indexes"); + path.push(&self.id); + Ok(Index::new(self.open_options, path)?) } } struct IndexView<'a, U> { txn: heed::RoTxn<'a>, - index: &'a Index, + index: Ref<'a, String, Index>, update_store: &'a U, } -struct IndexViewMut<'a, U> { - txn: heed::RwTxn<'a, 'a>, - index: &'a Index, - update_store: &'a U, -} - -impl<'a, U> Deref for IndexViewMut<'a, U> { - type Target = IndexView<'a, U>; - - fn deref(&self) -> &Self::Target { - IndexView { - txn: *self.txn, - index: self.index, - update_store: self.update_store, - } - } -} - impl<'a, U: UpdateStore> IndexView<'a, U> { pub fn search(&self, search_query: SearchQuery) -> Result { - let mut search = self.index.search(self.txn); + let mut search = self.index.search(&self.txn); if let Some(query) = &search_query.q { search.query(query); } @@ -115,15 +103,15 @@ impl<'a, U: UpdateStore> IndexView<'a, U> { } pub fn fields_ids_map(&self) -> Result { - self.index.fields_ids_map(self.txn) + Ok(self.index.fields_ids_map(&self.txn)?) } - pub fn fields_displayed_fields_ids(&self) -> Result { - self.index.fields_displayed_fields_ids(self.txn) + pub fn fields_displayed_fields_ids(&self) -> Result>> { + Ok(self.index.displayed_fields_ids(&self.txn)?) } - pub fn documents(&self, ids: &[u32]) -> Result> { - self.index.documents(self.txn, ids) + pub fn documents(&self, ids: Vec) -> Result)>> { + Ok(self.index.documents(&self.txn, ids)?) } } @@ -133,28 +121,29 @@ impl IndexController { pub fn new(path: impl AsRef, update_store: U) -> Result { // If index controller metadata is present, we return the env, otherwise, we create a new // metadata from scratch before returning a new env. - let env = match IndexControllerMeta::from_path(path)? { + let path = path.as_ref().to_path_buf(); + let env = match IndexControllerMeta::from_path(&path)? { Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, None => { let open_options = EnvOpenOptions::new() .map_size(page_size::get() * 1000); let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; let created_at = Utc::now(); - let meta = IndexControllerMeta { open_options, created_at }; + let meta = IndexControllerMeta { open_options: open_options.clone(), created_at }; meta.to_path(path)?; env } }; let indexes = DashMap::new(); - let indexes_db = match env.open_database(INDEXES_DB_NAME)? { + let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { Some(indexes_db) => indexes_db, - None => env.create_database(INDEXES_DB_NAME)?, + None => env.create_database(Some(INDEXES_DB_NAME))?, }; - Ok(Self { env, indexes, indexes_db, update_store }) + Ok(Self { env, indexes, indexes_db, update_store, path }) } - pub fn get_or_create>(&mut self, name: S) -> Result> { + pub fn get_or_create>(&mut self, name: S) -> Result> { todo!() } @@ -162,19 +151,26 @@ impl IndexController { /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check /// for metadata to launch the index. pub fn get>(&self, name: S) -> Result>> { + let update_store = &self.update_store; match self.indexes.get(name.as_ref()) { Some(index) => { let txn = index.read_txn()?; - let update_store = &self.update_store; Ok(Some(IndexView { index, update_store, txn })) } None => { let txn = self.env.read_txn()?; match self.indexes_db.get(&txn, name.as_ref())? { Some(meta) => { - let index = meta.open_index()?; + let index = meta.open_index(self.path)?; self.indexes.insert(name.as_ref().to_owned(), index); - Ok(self.indexes.get(name.as_ref())) + // TODO: create index view + match self.indexes.get(name.as_ref()) { + Some(index) => { + let txn = index.read_txn()?; + Ok(Some(IndexView { index, txn, update_store })) + } + None => Ok(None) + } } None => Ok(None) } @@ -182,7 +178,7 @@ impl IndexController { } } - pub fn get_mut>(&self, name: S) -> Result>> { + pub fn get_mut>(&self, name: S) -> Result>> { todo!() } diff --git a/src/updates/mod.rs b/src/updates/mod.rs index d92a3b16f..faefe7804 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -55,6 +55,8 @@ pub struct UpdateQueue { inner: Arc>, } +impl crate::index_controller::UpdateStore for UpdateQueue {} + impl Deref for UpdateQueue { type Target = Arc>; From 6a3f625e11bc9ab41a2bd01a306b3c8a63ea65cd Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 16 Jan 2021 15:09:48 +0100 Subject: [PATCH 019/522] WIP: refactor IndexController change the architecture of the index controller to allow it to own an index store. --- Cargo.lock | 40 ++ Cargo.toml | 1 + src/data/mod.rs | 47 +- src/data/search.rs | 4 +- src/data/updates.rs | 66 ++- src/index_controller/index_store.rs | 255 ++++++++++ src/index_controller/mod.rs | 293 +++++------- src/index_controller/update_store/mod.rs | 49 ++ src/lib.rs | 2 +- src/option.rs | 52 +- src/routes/document.rs | 2 +- src/routes/index.rs | 66 +-- src/routes/settings/mod.rs | 14 +- src/updates/mod.rs | 12 +- src/updates/update_store.rs | 581 +++++++++++++++++++++++ 15 files changed, 1197 insertions(+), 287 deletions(-) create mode 100644 src/index_controller/index_store.rs create mode 100644 src/index_controller/update_store/mod.rs create mode 100644 src/updates/update_store.rs diff --git a/Cargo.lock b/Cargo.lock index e0457b4d5..0282bfe92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,15 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "actix-codec" version = "0.3.0" @@ -1639,6 +1649,7 @@ dependencies = [ "mime", "obkv", "once_cell", + "ouroboros", "page_size", "rand 0.7.3", "rayon", @@ -1941,6 +1952,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ouroboros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069fb33e127cabdc8ad6a287eed9719b85c612d36199777f6dc41ad91f7be41a" +dependencies = [ + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad938cc920f299d6dce91e43d3ce316e785f4aa4bc4243555634dc2967098fc6" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "page_size" version = "0.4.2" @@ -2771,6 +2805,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "standback" version = "0.2.13" diff --git a/Cargo.toml b/Cargo.toml index c06847253..26a3dc1b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ whoami = "1.0.0" dashmap = "4.0.2" page_size = "0.4.2" obkv = "0.1.1" +ouroboros = "0.8.0" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 4600531bf..4258278d2 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -8,17 +8,16 @@ use std::sync::Arc; use sha2::Digest; -use crate::{option::Opt, updates::Settings}; -use crate::updates::UpdateQueue; -use crate::index_controller::IndexController; +use crate::{option::Opt, index_controller::Settings}; +use crate::index_controller::{IndexStore, UpdateStore}; #[derive(Clone)] pub struct Data { - inner: Arc, + inner: Arc>, } impl Deref for Data { - type Target = DataInner; + type Target = DataInner; fn deref(&self) -> &Self::Target { &self.inner @@ -26,8 +25,8 @@ impl Deref for Data { } #[derive(Clone)] -pub struct DataInner { - pub indexes: Arc>, +pub struct DataInner { + pub indexes: Arc, api_keys: ApiKeys, options: Opt, } @@ -58,11 +57,10 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { - let db_size = options.max_mdb_size.get_bytes() as usize; - let indexes = IndexController::new(&options.db_path)?; - let indexes = Arc::new(indexes); - - let update_queue = Arc::new(UpdateQueue::new(&options, indexes.clone())?); + let path = options.db_path.clone(); + let index_store = IndexStore::new(&path)?; + let index_controller = UpdateStore::new(index_store); + let indexes = Arc::new(index_controller); let mut api_keys = ApiKeys { master: options.clone().master_key, @@ -72,31 +70,28 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { indexes, options, update_queue, api_keys }; + let inner = DataInner { indexes, options, api_keys }; let inner = Arc::new(inner); Ok(Data { inner }) } - pub fn settings>(&self, _index: S) -> anyhow::Result { - let txn = self.indexes.env.read_txn()?; - let fields_map = self.indexes.fields_ids_map(&txn)?; - println!("fields_map: {:?}", fields_map); + pub fn settings>(&self, index_uid: S) -> anyhow::Result { + let index = self.indexes + .get(&index_uid)? + .ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?; - let displayed_attributes = self.indexes - .displayed_fields(&txn)? + let displayed_attributes = index + .displayed_fields()? .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); - let searchable_attributes = self.indexes - .searchable_fields(&txn)? - .map(|fields| fields - .into_iter() - .map(String::from) - .collect()) + let searchable_attributes = index + .searchable_fields()? + .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); - let faceted_attributes = self.indexes.faceted_fields(&txn)? + let faceted_attributes = index.faceted_fields()? .into_iter() .map(|(k, v)| (k, v.to_string())) .collect(); diff --git a/src/data/search.rs b/src/data/search.rs index 69029d8a9..c30345073 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -107,10 +107,10 @@ impl Data { pub fn search>(&self, index: S, search_query: SearchQuery) -> anyhow::Result { let start = Instant::now(); let index = self.indexes - .get(index)? + .get(&index)? .ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?; - let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(search_query)?; + let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(&search_query)?; let fields_ids_map = index.fields_ids_map()?; diff --git a/src/data/updates.rs b/src/data/updates.rs index 0654f6fd3..507223d41 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,18 +1,20 @@ use std::ops::Deref; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +//use milli::update_store::UpdateStatus; use async_compression::tokio_02::write::GzipEncoder; use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use milli::update_store::UpdateStatus; use super::Data; -use crate::updates::{UpdateMeta, UpdateResult, UpdateStatusResponse, Settings}; +use crate::index_controller::IndexController; +use crate::index_controller::{UpdateStatusResponse, Settings}; + impl Data { pub async fn add_documents( &self, - _index: S, + index: S, method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, @@ -20,7 +22,7 @@ impl Data { where B: Deref, E: std::error::Error + Send + Sync + 'static, - S: AsRef, + S: AsRef + Send + Sync + 'static, { let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; let file = tokio::fs::File::from_std(file?); @@ -37,43 +39,39 @@ impl Data { let file = file.into_std().await; let mmap = unsafe { memmap::Mmap::map(&file)? }; - let meta = UpdateMeta::DocumentsAddition { method, format }; - - let queue = self.update_queue.clone(); - let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &mmap[..])).await??; - + let indexes = self.indexes.clone(); + let update = tokio::task::spawn_blocking(move ||indexes.add_documents(index, method, format, &mmap[..])).await??; Ok(update.into()) } - pub async fn update_settings>( + pub async fn update_settings + Send + Sync + 'static>( &self, - _index: S, + index: S, settings: Settings ) -> anyhow::Result { - let meta = UpdateMeta::Settings(settings); - let queue = self.update_queue.clone(); - let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &[])).await??; + let indexes = self.indexes.clone(); + let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??; Ok(update.into()) } - #[inline] - pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result>> { - self.update_queue.get_update_status(uid) - } + //#[inline] + //pub fn get_update_status>(&self, _index: S, uid: u64) -> anyhow::Result>> { + //self.indexes.get_update_status(uid) + //} - pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { - let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { - let mut metas = processing - .map(UpdateStatus::from) - .into_iter() - .chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - .chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - .chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - .chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - .collect::>(); - metas.sort_by(|a, b| a.id().cmp(&b.id())); - Ok(metas) - })?; - Ok(result) - } + //pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { + //let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { + //let mut metas = processing + //.map(UpdateStatus::from) + //.into_iter() + //.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.collect::>(); + //metas.sort_by(|a, b| a.id().cmp(&b.id())); + //Ok(metas) + //})?; + //Ok(result) + //} } diff --git a/src/index_controller/index_store.rs b/src/index_controller/index_store.rs new file mode 100644 index 000000000..6ba6621c3 --- /dev/null +++ b/src/index_controller/index_store.rs @@ -0,0 +1,255 @@ +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::collections::HashMap; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use heed::types::{Str, SerdeBincode}; +use heed::{EnvOpenOptions, Env, Database}; +use milli::{Index, FieldsIdsMap, SearchResult, FieldId, facet::FacetType}; +use serde::{Serialize, Deserialize}; +use ouroboros::self_referencing; + +use crate::data::SearchQuery; + +const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; +const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; +const INDEXES_DB_NAME: &str = "indexes_db"; + + +#[derive(Debug, Serialize, Deserialize)] +struct IndexStoreMeta { + open_options: EnvOpenOptions, + created_at: DateTime, +} + +impl IndexStoreMeta { + fn from_path(path: impl AsRef) -> Result> { + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); + if path.exists() { + let mut file = File::open(path)?; + let mut buffer = Vec::new(); + let n = file.read_to_end(&mut buffer)?; + let meta: IndexStoreMeta = serde_json::from_slice(&buffer[..n])?; + Ok(Some(meta)) + } else { + Ok(None) + } + } + + fn to_path(self, path: impl AsRef) -> Result<()> { + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); + if path.exists() { + Err(anyhow::anyhow!("Index controller metadata already exists")) + } else { + let mut file = File::create(path)?; + let json = serde_json::to_vec(&self)?; + file.write_all(&json)?; + Ok(()) + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct IndexMetadata { + created_at: DateTime, + open_options: EnvOpenOptions, + uuid: String, +} + +impl IndexMetadata { + fn open_index(self, path: impl AsRef) -> Result { + // create a path in the form "db_path/indexes/index_id" + let mut path = path.as_ref().to_path_buf(); + path.push("indexes"); + path.push(&self.uuid); + Ok(Index::new(self.open_options, path)?) + } +} + + +#[self_referencing] +pub struct IndexView { + pub index: Arc, + #[borrows(index)] + #[covariant] + pub txn: heed::RoTxn<'this>, + uuid: String, +} + +impl IndexView { + pub fn search(&self, search_query: &SearchQuery) -> Result { + self.with(|this| { + let mut search = this.index.search(&this.txn); + if let Some(query) = &search_query.q { + search.query(query); + } + + if let Some(offset) = search_query.offset { + search.offset(offset); + } + + let limit = search_query.limit; + search.limit(limit); + + Ok(search.execute()?) + }) + } + + #[inline] + pub fn fields_ids_map(&self) -> Result { + self.with(|this| Ok(this.index.fields_ids_map(&this.txn)?)) + + } + + #[inline] + pub fn displayed_fields_ids(&self) -> Result>> { + self.with(|this| Ok(this.index.displayed_fields_ids(&this.txn)?)) + } + + #[inline] + pub fn displayed_fields(&self) -> Result>> { + self.with(|this| Ok(this.index + .displayed_fields(&this.txn)? + .map(|fields| fields.into_iter().map(String::from).collect()))) + } + + #[inline] + pub fn searchable_fields(&self) -> Result>> { + self.with(|this| Ok(this.index + .searchable_fields(&this.txn)? + .map(|fields| fields.into_iter().map(String::from).collect()))) + } + + #[inline] + pub fn faceted_fields(&self) -> Result> { + self.with(|this| Ok(this.index.faceted_fields(&this.txn)?)) + } + + pub fn documents(&self, ids: &[u32]) -> Result)>> { + let txn = self.borrow_txn(); + let index = self.borrow_index(); + Ok(index.documents(txn, ids.into_iter().copied())?) + } + + //pub async fn add_documents( + //&self, + //method: IndexDocumentsMethod, + //format: UpdateFormat, + //mut stream: impl futures::Stream> + Unpin, + //) -> anyhow::Result + //where + //B: Deref, + //E: std::error::Error + Send + Sync + 'static, + //{ + //let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; + //let file = tokio::fs::File::from_std(file?); + //let mut encoder = GzipEncoder::new(file); + + //while let Some(result) = stream.next().await { + //let bytes = &*result?; + //encoder.write_all(&bytes[..]).await?; + //} + + //encoder.shutdown().await?; + //let mut file = encoder.into_inner(); + //file.sync_all().await?; + //let file = file.into_std().await; + //let mmap = unsafe { memmap::Mmap::map(&file)? }; + + //let meta = UpdateMeta::DocumentsAddition { method, format }; + + //let index = self.index.clone(); + //let queue = self.update_store.clone(); + //let update = tokio::task::spawn_blocking(move || queue.register_update(index, meta, &mmap[..])).await??; + //Ok(update.into()) + //} +} + +pub struct IndexStore { + path: PathBuf, + env: Env, + indexes_db: Database>, + indexes: DashMap)>, +} + +impl IndexStore { + /// Open the index controller from meta found at path, and create a new one if no meta is + /// found. + pub fn new(path: impl AsRef) -> Result { + // If index controller metadata is present, we return the env, otherwise, we create a new + // metadata from scratch before returning a new env. + let path = path.as_ref().to_path_buf(); + let env = match IndexStoreMeta::from_path(&path)? { + Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, + None => { + let mut open_options = EnvOpenOptions::new(); + open_options.map_size(page_size::get() * 1000); + let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; + let created_at = Utc::now(); + let meta = IndexStoreMeta { open_options: open_options.clone(), created_at }; + meta.to_path(&path)?; + env + } + }; + let indexes = DashMap::new(); + let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { + Some(indexes_db) => indexes_db, + None => env.create_database(Some(INDEXES_DB_NAME))?, + }; + + Ok(Self { env, indexes, indexes_db, path }) + } + + pub fn get_or_create>(&self, _name: S) -> Result { + todo!() + } + + /// Get an index with read access to the db. The index are lazily loaded, meaning that we first + /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check + /// for metadata to launch the index. + pub fn get>(&self, name: S) -> Result> { + match self.indexes.get(name.as_ref()) { + Some(entry) => { + let index = entry.1.clone(); + let uuid = entry.0.clone(); + let view = IndexView::try_new(index, |index| index.read_txn(), uuid)?; + Ok(Some(view)) + } + None => { + let txn = self.env.read_txn()?; + match self.indexes_db.get(&txn, name.as_ref())? { + Some(meta) => { + let uuid = meta.uuid.clone(); + let index = Arc::new(meta.open_index(&self.path)?); + self.indexes.insert(name.as_ref().to_owned(), (uuid.clone(), index.clone())); + let view = IndexView::try_new(index, |index| index.read_txn(), uuid)?; + Ok(Some(view)) + } + None => Ok(None) + } + } + } + } + + pub fn get_mut>(&self, _name: S) -> Result> { + todo!() + } + + pub async fn delete_index>(&self, _name:S) -> Result<()> { + todo!() + } + + pub async fn list_indices(&self) -> Result> { + todo!() + } + + pub async fn rename_index(&self, _old: &str, _new: &str) -> Result<()> { + todo!() + } +} diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index a3b29879a..c927a6c5b 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,196 +1,145 @@ -use std::fs::File; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; +mod index_store; +mod update_store; + +pub use index_store::IndexStore; +pub use update_store::UpdateStore; + +use std::num::NonZeroUsize; +use std::ops::Deref; +use std::collections::HashMap; use anyhow::Result; -use chrono::{DateTime, Utc}; -use dashmap::DashMap; -use dashmap::mapref::one::Ref; -use heed::types::{Str, SerdeBincode}; -use heed::{EnvOpenOptions, Env, Database}; -use milli::{Index, FieldsIdsMap, SearchResult, FieldId}; -use serde::{Serialize, Deserialize}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use milli::update_store::{Processed, Processing, Failed, Pending, Aborted}; +use serde::{Serialize, Deserialize, de::Deserializer}; -use crate::data::SearchQuery; +pub type UpdateStatusResponse = UpdateStatus; -const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; -const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; -const INDEXES_DB_NAME: &str = "indexes_db"; - -pub trait UpdateStore {} - -pub struct IndexController { - path: PathBuf, - update_store: U, - env: Env, - indexes_db: Database>, - indexes: DashMap, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum UpdateMeta { + DocumentsAddition { method: IndexDocumentsMethod, format: UpdateFormat }, + ClearDocuments, + Settings(Settings), + Facets(Facets), } -#[derive(Debug, Serialize, Deserialize)] -struct IndexControllerMeta { - open_options: EnvOpenOptions, - created_at: DateTime, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Facets { + pub level_group_size: Option, + pub min_level_size: Option, } -impl IndexControllerMeta { - fn from_path(path: impl AsRef) -> Result> { - let mut path = path.as_ref().to_path_buf(); - path.push(CONTROLLER_META_FILENAME); - if path.exists() { - let mut file = File::open(path)?; - let mut buffer = Vec::new(); - let n = file.read_to_end(&mut buffer)?; - let meta: IndexControllerMeta = serde_json::from_slice(&buffer[..n])?; - Ok(Some(meta)) - } else { - Ok(None) - } - } +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "type")] +pub enum UpdateStatus { + Pending { update_id: u64, meta: Pending }, + Progressing { update_id: u64, meta: P }, + Processed { update_id: u64, meta: Processed }, + Aborted { update_id: u64, meta: Aborted }, +} - fn to_path(self, path: impl AsRef) -> Result<()> { - let mut path = path.as_ref().to_path_buf(); - path.push(CONTROLLER_META_FILENAME); - if path.exists() { - Err(anyhow::anyhow!("Index controller metadata already exists")) - } else { - let mut file = File::create(path)?; - let json = serde_json::to_vec(&self)?; - file.write_all(&json)?; - Ok(()) +fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> +where T: Deserialize<'de>, + D: Deserializer<'de> +{ + Deserialize::deserialize(deserializer).map(Some) +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Settings { + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub displayed_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub searchable_attributes: Option>>, + + #[serde(default)] + pub faceted_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub criteria: Option>>, +} + +impl Settings { + pub fn cleared() -> Self { + Self { + displayed_attributes: Some(None), + searchable_attributes: Some(None), + faceted_attributes: Some(None), + criteria: Some(None), } } } - -#[derive(Debug, Serialize, Deserialize)] -struct IndexMetadata { - created_at: DateTime, - open_options: EnvOpenOptions, - id: String, +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum UpdateResult { + //DocumentsAddition(DocumentAdditionResult), + Other, } -impl IndexMetadata { - fn open_index(&self, path: impl AsRef) -> Result { - // create a path in the form "db_path/indexes/index_id" - let mut path = path.as_ref().to_path_buf(); - path.push("indexes"); - path.push(&self.id); - Ok(Index::new(self.open_options, path)?) - } -} +/// The `IndexController` is in charge of the access to the underlying indices. It splits the logic +/// for read access which is provided, and write access which must be provided. This allows the +/// implementer to define the behaviour of write accesses to the indices, and abstract the +/// scheduling of the updates. The implementer must be able to provide an instance of `IndexStore` +pub trait IndexController: Deref { -struct IndexView<'a, U> { - txn: heed::RoTxn<'a>, - index: Ref<'a, String, Index>, - update_store: &'a U, -} + /* + * Write operations + * + * Logic for the write operation need to be provided by the implementer, since they can be made + * asynchronous thanks to an update_store for example. + * + * */ -impl<'a, U: UpdateStore> IndexView<'a, U> { - pub fn search(&self, search_query: SearchQuery) -> Result { - let mut search = self.index.search(&self.txn); - if let Some(query) = &search_query.q { - search.query(query); - } + /// Perform document addition on the database. If the provided index does not exist, it will be + /// created when the addition is applied to the index. + fn add_documents>( + &self, + index: S, + method: IndexDocumentsMethod, + format: UpdateFormat, + data: &[u8], + ) -> anyhow::Result; - if let Some(offset) = search_query.offset { - search.offset(offset); - } + /// Updates an index settings. If the index does not exist, it will be created when the update + /// is applied to the index. + fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; - let limit = search_query.limit; - search.limit(limit); + /// Create an index with the given `index_uid`. + fn create_index>(&self, index_uid: S) -> Result<()>; - Ok(search.execute()?) - } + /// Delete index with the given `index_uid`, attempting to close it beforehand. + fn delete_index>(&self, index_uid: S) -> Result<()>; - pub fn fields_ids_map(&self) -> Result { - Ok(self.index.fields_ids_map(&self.txn)?) - } + /// Swap two indexes, concretely, it simply swaps the index the names point to. + fn swap_indices, S2: AsRef>(&self, index1_uid: S1, index2_uid: S2) -> Result<()>; - pub fn fields_displayed_fields_ids(&self) -> Result>> { - Ok(self.index.displayed_fields_ids(&self.txn)?) - } - - pub fn documents(&self, ids: Vec) -> Result)>> { - Ok(self.index.documents(&self.txn, ids)?) - } -} - -impl IndexController { - /// Open the index controller from meta found at path, and create a new one if no meta is - /// found. - pub fn new(path: impl AsRef, update_store: U) -> Result { - // If index controller metadata is present, we return the env, otherwise, we create a new - // metadata from scratch before returning a new env. - let path = path.as_ref().to_path_buf(); - let env = match IndexControllerMeta::from_path(&path)? { - Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, - None => { - let open_options = EnvOpenOptions::new() - .map_size(page_size::get() * 1000); - let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; - let created_at = Utc::now(); - let meta = IndexControllerMeta { open_options: open_options.clone(), created_at }; - meta.to_path(path)?; - env - } - }; - let indexes = DashMap::new(); - let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { - Some(indexes_db) => indexes_db, - None => env.create_database(Some(INDEXES_DB_NAME))?, - }; - - Ok(Self { env, indexes, indexes_db, update_store, path }) - } - - pub fn get_or_create>(&mut self, name: S) -> Result> { - todo!() - } - - /// Get an index with read access to the db. The index are lazily loaded, meaning that we first - /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check - /// for metadata to launch the index. - pub fn get>(&self, name: S) -> Result>> { - let update_store = &self.update_store; - match self.indexes.get(name.as_ref()) { - Some(index) => { - let txn = index.read_txn()?; - Ok(Some(IndexView { index, update_store, txn })) - } - None => { - let txn = self.env.read_txn()?; - match self.indexes_db.get(&txn, name.as_ref())? { - Some(meta) => { - let index = meta.open_index(self.path)?; - self.indexes.insert(name.as_ref().to_owned(), index); - // TODO: create index view - match self.indexes.get(name.as_ref()) { - Some(index) => { - let txn = index.read_txn()?; - Ok(Some(IndexView { index, txn, update_store })) - } - None => Ok(None) - } - } - None => Ok(None) - } - } - } - } - - pub fn get_mut>(&self, name: S) -> Result>> { - todo!() - } - - pub async fn delete_index>(&self, name:S) -> Result<()> { - todo!() - } - - pub async fn list_indices(&self) -> Result> { - todo!() - } - - pub async fn rename_index(&self, old: &str, new: &str) -> Result<()> { + /// Apply an update to the given index. This method can be called when an update is ready to be + /// processed + fn handle_update>( + &self, + _index: S, + _update_id: u64, + _meta: Processing, + _content: &[u8] + ) -> Result, Failed> { todo!() } } + diff --git a/src/index_controller/update_store/mod.rs b/src/index_controller/update_store/mod.rs new file mode 100644 index 000000000..84db2f63d --- /dev/null +++ b/src/index_controller/update_store/mod.rs @@ -0,0 +1,49 @@ +use std::ops::Deref; + +use super::{IndexStore, IndexController}; + +pub struct UpdateStore { + index_store: IndexStore, +} + +impl Deref for UpdateStore { + type Target = IndexStore; + + fn deref(&self) -> &Self::Target { + &self.index_store + } +} + +impl UpdateStore { + pub fn new(index_store: IndexStore) -> Self { + Self { index_store } + } +} + +impl IndexController for UpdateStore { + fn add_documents>( + &self, + _index: S, + _method: milli::update::IndexDocumentsMethod, + _format: milli::update::UpdateFormat, + _data: &[u8], + ) -> anyhow::Result { + todo!() + } + + fn update_settings>(&self, _index_uid: S, _settings: crate::index_controller::Settings) -> anyhow::Result { + todo!() + } + + fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { + todo!() + } + + fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { + todo!() + } + + fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { + todo!() + } +} diff --git a/src/lib.rs b/src/lib.rs index f5dd79b0b..df9381914 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,7 @@ pub mod error; pub mod helpers; pub mod option; pub mod routes; -mod updates; +//mod updates; mod index_controller; use actix_http::Error; diff --git a/src/option.rs b/src/option.rs index f9e98f4fa..f280553e1 100644 --- a/src/option.rs +++ b/src/option.rs @@ -9,10 +9,60 @@ use rustls::{ AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, RootCertStore, }; +use grenad::CompressionType; use structopt::StructOpt; -use crate::updates::IndexerOpts; +#[derive(Debug, Clone, StructOpt)] +pub struct IndexerOpts { + /// The amount of documents to skip before printing + /// a log regarding the indexing advancement. + #[structopt(long, default_value = "100000")] // 100k + pub log_every_n: usize, + /// MTBL max number of chunks in bytes. + #[structopt(long)] + pub max_nb_chunks: Option, + + /// The maximum amount of memory to use for the MTBL buffer. It is recommended + /// to use something like 80%-90% of the available memory. + /// + /// It is automatically split by the number of jobs e.g. if you use 7 jobs + /// and 7 GB of max memory, each thread will use a maximum of 1 GB. + #[structopt(long, default_value = "7 GiB")] + pub max_memory: Byte, + + /// Size of the linked hash map cache when indexing. + /// The bigger it is, the faster the indexing is but the more memory it takes. + #[structopt(long, default_value = "500")] + pub linked_hash_map_size: usize, + + /// The name of the compression algorithm to use when compressing intermediate + /// chunks during indexing documents. + /// + /// Choosing a fast algorithm will make the indexing faster but may consume more memory. + #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] + pub chunk_compression_type: CompressionType, + + /// The level of compression of the chosen algorithm. + #[structopt(long, requires = "chunk-compression-type")] + pub chunk_compression_level: Option, + + /// The number of bytes to remove from the begining of the chunks while reading/sorting + /// or merging them. + /// + /// File fusing must only be enable on file systems that support the `FALLOC_FL_COLLAPSE_RANGE`, + /// (i.e. ext4 and XFS). File fusing will only work if the `enable-chunk-fusing` is set. + #[structopt(long, default_value = "4 GiB")] + pub chunk_fusing_shrink_size: Byte, + + /// Enable the chunk fusing or not, this reduces the amount of disk used by a factor of 2. + #[structopt(long)] + pub enable_chunk_fusing: bool, + + /// Number of parallel jobs for indexing, defaults to # of CPUs. + #[structopt(long)] + pub indexing_jobs: Option, +} const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; #[derive(Debug, Clone, StructOpt)] diff --git a/src/routes/document.rs b/src/routes/document.rs index 6c5f93991..aeec0e5df 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -122,7 +122,7 @@ async fn add_documents_json( ) -> Result { let addition_result = data .add_documents( - &path.index_uid, + path.into_inner().index_uid, IndexDocumentsMethod::UpdateDocuments, UpdateFormat::Json, body diff --git a/src/routes/index.rs b/src/routes/index.rs index 515e771e1..fa4ae0679 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -1,7 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; -use log::error; +//use log::error; use serde::{Deserialize, Serialize}; use crate::Data; @@ -94,8 +94,8 @@ async fn delete_index( #[derive(Deserialize)] struct UpdateParam { - index_uid: String, - update_id: u64, + _index_uid: String, + _update_id: u64, } #[get( @@ -103,39 +103,41 @@ struct UpdateParam { wrap = "Authentication::Private" )] async fn get_update_status( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { - let result = data.get_update_status(&path.index_uid, path.update_id); - match result { - Ok(Some(meta)) => { - let json = serde_json::to_string(&meta).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Ok(None) => { - todo!() - } - Err(e) => { - error!("{}", e); - todo!() - } - } + todo!() + //let result = data.get_update_status(&path.index_uid, path.update_id); + //match result { + //Ok(Some(meta)) => { + //let json = serde_json::to_string(&meta).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Ok(None) => { + //todo!() + //} + //Err(e) => { + //error!("{}", e); + //todo!() + //} + //} } #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] async fn get_all_updates_status( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { - let result = data.get_updates_status(&path.index_uid); - match result { - Ok(metas) => { - let json = serde_json::to_string(&metas).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Err(e) => { - error!("{}", e); - todo!() - } - } + todo!() + //let result = data.get_updates_status(&path.index_uid); + //match result { + //Ok(metas) => { + //let json = serde_json::to_string(&metas).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Err(e) => { + //error!("{}", e); + //todo!() + //} + //} } diff --git a/src/routes/settings/mod.rs b/src/routes/settings/mod.rs index d54ad469c..56c1d34a0 100644 --- a/src/routes/settings/mod.rs +++ b/src/routes/settings/mod.rs @@ -3,7 +3,7 @@ use log::error; use crate::Data; use crate::error::ResponseError; -use crate::updates::Settings; +use crate::index_controller::Settings; use crate::helpers::Authentication; #[macro_export] @@ -15,19 +15,19 @@ macro_rules! make_setting_route { use crate::data; use crate::error::ResponseError; use crate::helpers::Authentication; - use crate::updates::Settings; + use crate::index_controller::Settings; #[actix_web::delete($route, wrap = "Authentication::Private")] pub async fn delete( data: web::Data, index_uid: web::Path, ) -> Result { - use crate::updates::Settings; + use crate::index_controller::Settings; let settings = Settings { $attr: Some(None), ..Default::default() }; - match data.update_settings(index_uid.as_ref(), settings).await { + match data.update_settings(index_uid.into_inner(), settings).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -50,7 +50,7 @@ macro_rules! make_setting_route { ..Default::default() }; - match data.update_settings(index_uid.as_ref(), settings).await { + match data.update_settings(index_uid.into_inner(), settings).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -141,7 +141,7 @@ async fn update_all( index_uid: web::Path, body: web::Json, ) -> Result { - match data.update_settings(index_uid.as_ref(), body.into_inner()).await { + match data.update_settings(index_uid.into_inner(), body.into_inner()).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -176,7 +176,7 @@ async fn delete_all( index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); - match data.update_settings(index_uid.as_ref(), settings).await { + match data.update_settings(index_uid.into_inner(), settings).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) diff --git a/src/updates/mod.rs b/src/updates/mod.rs index faefe7804..7e0802595 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -1,10 +1,10 @@ mod settings; +mod update_store; pub use settings::{Settings, Facets}; use std::io; use std::sync::Arc; -use std::ops::Deref; use std::fs::create_dir_all; use std::collections::HashMap; @@ -55,16 +55,6 @@ pub struct UpdateQueue { inner: Arc>, } -impl crate::index_controller::UpdateStore for UpdateQueue {} - -impl Deref for UpdateQueue { - type Target = Arc>; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - #[derive(Debug, Clone, StructOpt)] pub struct IndexerOpts { /// The amount of documents to skip before printing diff --git a/src/updates/update_store.rs b/src/updates/update_store.rs new file mode 100644 index 000000000..f750fc38b --- /dev/null +++ b/src/updates/update_store.rs @@ -0,0 +1,581 @@ +use std::path::Path; +use std::sync::{Arc, RwLock}; + +use crossbeam_channel::Sender; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::{EnvOpenOptions, Env, Database}; +use serde::{Serialize, Deserialize}; +use chrono::{DateTime, Utc}; + +type BEU64 = heed::zerocopy::U64; + +#[derive(Clone)] +pub struct UpdateStore { + env: Env, + pending_meta: Database, SerdeJson>>, + pending: Database, ByteSlice>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, + notification_sender: Sender<()>, +} + +pub trait UpdateHandler { + fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed>; +} + +impl UpdateHandler for F +where F: FnMut(u64, Processing, &[u8]) -> Result, Failed> + Send + 'static { + fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed> { + self(update_id, meta, content) + } +} + +impl UpdateStore { + pub fn open( + size: Option, + path: P, + mut update_handler: U, + ) -> heed::Result>> + where + P: AsRef, + U: UpdateHandler + Send + 'static, + M: for<'a> Deserialize<'a> + Serialize + Send + Sync + Clone, + N: Serialize, + E: Serialize, + { + let mut options = EnvOpenOptions::new(); + if let Some(size) = size { + options.map_size(size); + } + options.max_dbs(5); + + let env = options.open(path)?; + let pending_meta = env.create_database(Some("pending-meta"))?; + let pending = env.create_database(Some("pending"))?; + let processed_meta = env.create_database(Some("processed-meta"))?; + let aborted_meta = env.create_database(Some("aborted-meta"))?; + let failed_meta = env.create_database(Some("failed-meta"))?; + let processing = Arc::new(RwLock::new(None)); + + let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + // Send a first notification to trigger the process. + let _ = notification_sender.send(()); + + let update_store = Arc::new(UpdateStore { + env, + pending, + pending_meta, + processed_meta, + aborted_meta, + notification_sender, + failed_meta, + processing, + }); + + let update_store_cloned = update_store.clone(); + std::thread::spawn(move || { + // Block and wait for something to process. + for () in notification_receiver { + loop { + match update_store_cloned.process_pending_update(&mut update_handler) { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => eprintln!("error while processing update: {}", e), + } + } + } + }); + + Ok(update_store) + } + + /// Returns the new biggest id to use to store the new update. + fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { + let last_pending = self.pending_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_processed = self.processed_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_aborted = self.aborted_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_update_id = [last_pending, last_processed, last_aborted] + .iter() + .copied() + .flatten() + .max(); + + match last_update_id { + Some(last_id) => Ok(last_id + 1), + None => Ok(0), + } + } + + /// Registers the update content in the pending store and the meta + /// into the pending-meta store. Returns the new unique update id. + pub fn register_update(&self, meta: M, content: &[u8]) -> heed::Result> + where M: Serialize, + { + let mut wtxn = self.env.write_txn()?; + + // We ask the update store to give us a new update id, this is safe, + // no other update can have the same id because we use a write txn before + // asking for the id and registering it so other update registering + // will be forced to wait for a new write txn. + let update_id = self.new_update_id(&wtxn)?; + let update_key = BEU64::new(update_id); + + let meta = Pending::new(meta, update_id); + self.pending_meta.put(&mut wtxn, &update_key, &meta)?; + self.pending.put(&mut wtxn, &update_key, content)?; + + wtxn.commit()?; + + if let Err(e) = self.notification_sender.try_send(()) { + assert!(!e.is_disconnected(), "update notification channel is disconnected"); + } + Ok(meta) + } + /// Executes the user provided function on the next pending update (the one with the lowest id). + /// This is asynchronous as it let the user process the update with a read-only txn and + /// only writing the result meta to the processed-meta store *after* it has been processed. + fn process_pending_update(&self, handler: &mut U) -> heed::Result> + where + U: UpdateHandler, + M: for<'a> Deserialize<'a> + Serialize + Clone, + N: Serialize, + E: Serialize, + { + // Create a read transaction to be able to retrieve the pending update in order. + let rtxn = self.env.read_txn()?; + let first_meta = self.pending_meta.first(&rtxn)?; + + // If there is a pending update we process and only keep + // a reader while processing it, not a writer. + match first_meta { + Some((first_id, pending)) => { + let first_content = self.pending + .get(&rtxn, &first_id)? + .expect("associated update content"); + + // we cahnge the state of the update from pending to processing before we pass it + // to the update handler. Processing store is non persistent to be able recover + // from a failure + let processing = pending.processing(); + self.processing + .write() + .unwrap() + .replace(processing.clone()); + // Process the pending update using the provided user function. + let result = handler.handle_update(first_id.get(), processing, first_content); + drop(rtxn); + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.processing + .write() + .unwrap() + .take(); + self.pending_meta.delete(&mut wtxn, &first_id)?; + self.pending.delete(&mut wtxn, &first_id)?; + match result { + Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, + Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, + } + wtxn.commit()?; + + Ok(Some(())) + }, + None => Ok(None) + } + } + + /// The id and metadata of the update that is currently being processed, + /// `None` if no update is being processed. + pub fn processing_update(&self) -> heed::Result)>> + where M: for<'a> Deserialize<'a>, + { + let rtxn = self.env.read_txn()?; + match self.pending_meta.first(&rtxn)? { + Some((key, meta)) => Ok(Some((key.get(), meta))), + None => Ok(None), + } + } + + /// Execute the user defined function with the meta-store iterators, the first + /// iterator is the *processed* meta one, the second the *aborted* meta one + /// and, the last is the *pending* meta one. + pub fn iter_metas(&self, mut f: F) -> heed::Result + where + M: for<'a> Deserialize<'a> + Clone, + N: for<'a> Deserialize<'a>, + F: for<'a> FnMut( + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + ) -> heed::Result, + { + let rtxn = self.env.read_txn()?; + + // We get the pending, processed and aborted meta iterators. + let processed_iter = self.processed_meta.iter(&rtxn)?; + let aborted_iter = self.aborted_meta.iter(&rtxn)?; + let pending_iter = self.pending_meta.iter(&rtxn)?; + let processing = self.processing.read().unwrap().clone(); + let failed_iter = self.failed_meta.iter(&rtxn)?; + + // We execute the user defined function with both iterators. + (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + } + + /// Returns the update associated meta or `None` if the update doesn't exist. + pub fn meta(&self, update_id: u64) -> heed::Result>> + where + M: for<'a> Deserialize<'a> + Clone, + N: for<'a> Deserialize<'a>, + E: for<'a> Deserialize<'a>, + { + let rtxn = self.env.read_txn()?; + let key = BEU64::new(update_id); + + if let Some(ref meta) = *self.processing.read().unwrap() { + if meta.id() == update_id { + return Ok(Some(UpdateStatus::Processing(meta.clone()))); + } + } + + println!("pending"); + if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Pending(meta))); + } + + println!("processed"); + if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Processed(meta))); + } + + if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Aborted(meta))); + } + + if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Failed(meta))); + } + + Ok(None) + } + + /// Aborts an update, an aborted update content is deleted and + /// the meta of it is moved into the aborted updates database. + /// + /// Trying to abort an update that is currently being processed, an update + /// that as already been processed or which doesn't actually exist, will + /// return `None`. + pub fn abort_update(&self, update_id: u64) -> heed::Result>> + where M: Serialize + for<'a> Deserialize<'a>, + { + let mut wtxn = self.env.write_txn()?; + let key = BEU64::new(update_id); + + // We cannot abort an update that is currently being processed. + if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { + return Ok(None); + } + + let pending = match self.pending_meta.get(&wtxn, &key)? { + Some(meta) => meta, + None => return Ok(None), + }; + + let aborted = pending.abort(); + + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + + wtxn.commit()?; + + Ok(Some(aborted)) + } + + /// Aborts all the pending updates, and not the one being currently processed. + /// Returns the update metas and ids that were successfully aborted. + pub fn abort_pendings(&self) -> heed::Result)>> + where M: Serialize + for<'a> Deserialize<'a>, + { + let mut wtxn = self.env.write_txn()?; + let mut aborted_updates = Vec::new(); + + // We skip the first pending update as it is currently being processed. + for result in self.pending_meta.iter(&wtxn)?.skip(1) { + let (key, pending) = result?; + let id = key.get(); + aborted_updates.push((id, pending.abort())); + } + + for (id, aborted) in &aborted_updates { + let key = BEU64::new(*id); + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + } + + wtxn.commit()?; + + Ok(aborted_updates) + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Pending { + update_id: u64, + meta: M, + enqueued_at: DateTime, +} + +impl Pending { + fn new(meta: M, update_id: u64) -> Self { + Self { + enqueued_at: Utc::now(), + meta, + update_id, + } + } + + pub fn processing(self) -> Processing { + Processing { + from: self, + started_processing_at: Utc::now(), + } + } + + pub fn abort(self) -> Aborted { + Aborted { + from: self, + aborted_at: Utc::now(), + } + } + + pub fn meta(&self) -> &M { + &self.meta + } + + pub fn id(&self) -> u64 { + self.update_id + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Processed { + success: N, + processed_at: DateTime, + #[serde(flatten)] + from: Processing, +} + +impl Processed { + fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Processing { + #[serde(flatten)] + from: Pending, + started_processing_at: DateTime, +} + +impl Processing { + pub fn id(&self) -> u64 { + self.from.id() + } + + pub fn meta(&self) -> &M { + self.from.meta() + } + + pub fn process(self, meta: N) -> Processed { + Processed { + success: meta, + from: self, + processed_at: Utc::now(), + } + } + + pub fn fail(self, error: E) -> Failed { + Failed { + from: self, + error, + failed_at: Utc::now(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Aborted { + #[serde(flatten)] + from: Pending, + aborted_at: DateTime, +} + +impl Aborted { + fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Failed { + #[serde(flatten)] + from: Processing, + error: E, + failed_at: DateTime, +} + +impl Failed { + fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize)] +#[serde(tag = "status")] +pub enum UpdateStatus { + Processing(Processing), + Pending(Pending), + Processed(Processed), + Aborted(Aborted), + Failed(Failed), +} + +impl UpdateStatus { + pub fn id(&self) -> u64 { + match self { + UpdateStatus::Processing(u) => u.id(), + UpdateStatus::Pending(u) => u.id(), + UpdateStatus::Processed(u) => u.id(), + UpdateStatus::Aborted(u) => u.id(), + UpdateStatus::Failed(u) => u.id(), + } + } +} + +impl From> for UpdateStatus { + fn from(other: Pending) -> Self { + Self::Pending(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Aborted) -> Self { + Self::Aborted(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Processed) -> Self { + Self::Processed(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Processing) -> Self { + Self::Processing(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Failed) -> Self { + Self::Failed(other) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn simple() { + let dir = tempfile::tempdir().unwrap(); + let update_store = UpdateStore::open(None, dir, |_id, meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let meta = String::from("kiki"); + let update = update_store.register_update(meta, &[]).unwrap(); + thread::sleep(Duration::from_millis(100)); + let meta = update_store.meta(update.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + } + + #[test] + #[ignore] + fn long_running_update() { + let dir = tempfile::tempdir().unwrap(); + let update_store = UpdateStore::open(None, dir, |_id, meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let before_register = Instant::now(); + + let meta = String::from("kiki"); + let update_kiki = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("coco"); + let update_coco = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("cucu"); + let update_cucu = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + thread::sleep(Duration::from_millis(400 * 3 + 100)); + + let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "coco processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "cucu processed"); + } else { + panic!() + } + } +} From 74410d8c6b0e20c79df179a26f228ad058b93e3e Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 14:12:34 +0100 Subject: [PATCH 020/522] architecture rework --- Cargo.lock | 24 +- Cargo.toml | 1 + src/data/mod.rs | 31 +- src/data/search.rs | 107 +++--- src/data/updates.rs | 11 +- src/index_controller/_update_store/mod.rs | 17 + src/index_controller/index_store.rs | 181 ---------- .../local_index_controller/index_store.rs | 188 +++++++++++ .../local_index_controller/mod.rs | 57 ++++ .../local_index_controller/update_handler.rs | 206 ++++++++++++ .../local_index_controller/update_store.rs | 311 ++++++++++++++++++ src/index_controller/mod.rs | 25 +- src/index_controller/update_store/mod.rs | 49 --- src/index_controller/updates.rs | 167 ++++++++++ 14 files changed, 1065 insertions(+), 310 deletions(-) create mode 100644 src/index_controller/_update_store/mod.rs create mode 100644 src/index_controller/local_index_controller/index_store.rs create mode 100644 src/index_controller/local_index_controller/mod.rs create mode 100644 src/index_controller/local_index_controller/update_handler.rs create mode 100644 src/index_controller/local_index_controller/update_store.rs delete mode 100644 src/index_controller/update_store/mod.rs create mode 100644 src/index_controller/updates.rs diff --git a/Cargo.lock b/Cargo.lock index 0282bfe92..5ceaef691 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1137,6 +1137,17 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + [[package]] name = "gimli" version = "0.23.0" @@ -1608,7 +1619,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.18.0" +version = "0.18.1" dependencies = [ "actix-http", ] @@ -1669,6 +1680,7 @@ dependencies = [ "tempfile", "tokio", "ureq", + "uuid", "vergen", "walkdir", "whoami", @@ -2263,7 +2275,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.15", "libc", "rand_chacha", "rand_core 0.5.1", @@ -2302,7 +2314,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.15", ] [[package]] @@ -3365,11 +3377,11 @@ checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" [[package]] name = "uuid" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "rand 0.7.3", + "getrandom 0.2.2", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 26a3dc1b8..eb2fca43d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ dashmap = "4.0.2" page_size = "0.4.2" obkv = "0.1.1" ouroboros = "0.8.0" +uuid = "0.8.2" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 4258278d2..7494792bc 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -5,19 +5,20 @@ pub use search::{SearchQuery, SearchResult}; use std::ops::Deref; use std::sync::Arc; +use std::fs::create_dir_all; use sha2::Digest; use crate::{option::Opt, index_controller::Settings}; -use crate::index_controller::{IndexStore, UpdateStore}; +use crate::index_controller::{IndexController, LocalIndexController}; #[derive(Clone)] pub struct Data { - inner: Arc>, + inner: Arc, } impl Deref for Data { - type Target = DataInner; + type Target = DataInner; fn deref(&self) -> &Self::Target { &self.inner @@ -25,8 +26,8 @@ impl Deref for Data { } #[derive(Clone)] -pub struct DataInner { - pub indexes: Arc, +pub struct DataInner { + pub index_controller: Arc, api_keys: ApiKeys, options: Opt, } @@ -58,8 +59,9 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - let index_store = IndexStore::new(&path)?; - let index_controller = UpdateStore::new(index_store); + let indexer_opts = options.indexer_options.clone(); + create_dir_all(&path)?; + let index_controller = LocalIndexController::new(&path, indexer_opts)?; let indexes = Arc::new(index_controller); let mut api_keys = ApiKeys { @@ -70,28 +72,31 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { indexes, options, api_keys }; + let inner = DataInner { index_controller: indexes, options, api_keys }; let inner = Arc::new(inner); Ok(Data { inner }) } pub fn settings>(&self, index_uid: S) -> anyhow::Result { - let index = self.indexes - .get(&index_uid)? + let index = self.index_controller + .index(&index_uid)? .ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?; + let txn = index.read_txn()?; + let displayed_attributes = index - .displayed_fields()? + .displayed_fields(&txn)? .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); let searchable_attributes = index - .searchable_fields()? + .searchable_fields(&txn)? .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); - let faceted_attributes = index.faceted_fields()? + let faceted_attributes = index + .faceted_fields(&txn)? .into_iter() .map(|(k, v)| (k, v.to_string())) .collect(); diff --git a/src/data/search.rs b/src/data/search.rs index c30345073..246e3bdac 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -4,11 +4,11 @@ use std::time::Instant; use serde_json::{Value, Map}; use serde::{Deserialize, Serialize}; -use milli::{SearchResult as Results, obkv_to_json}; +use milli::{Index, obkv_to_json, FacetCondition}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use anyhow::bail; -use crate::error::Error; - +use crate::index_controller::IndexController; use super::Data; const DEFAULT_SEARCH_LIMIT: usize = 20; @@ -26,11 +26,68 @@ pub struct SearchQuery { pub attributes_to_retrieve: Option>, pub attributes_to_crop: Option>, pub crop_length: Option, - pub attributes_to_highlight: Option>, + pub attributes_to_highlight: Option>, pub filters: Option, pub matches: Option, pub facet_filters: Option, pub facets_distribution: Option>, + pub facet_condition: Option, +} + +impl SearchQuery { + pub fn perform(&self, index: impl AsRef) -> anyhow::Result{ + let index = index.as_ref(); + + let before_search = Instant::now(); + let rtxn = index.read_txn().unwrap(); + + let mut search = index.search(&rtxn); + + if let Some(ref query) = self.q { + search.query(query); + } + + if let Some(ref condition) = self.facet_condition { + if !condition.trim().is_empty() { + let condition = FacetCondition::from_str(&rtxn, &index, &condition).unwrap(); + search.facet_condition(condition); + } + } + + if let Some(offset) = self.offset { + search.offset(offset); + } + + let milli::SearchResult { documents_ids, found_words, nb_hits, limit, } = search.execute()?; + + let mut documents = Vec::new(); + let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields = match index.displayed_fields_ids(&rtxn).unwrap() { + Some(fields) => fields, + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new(&stop_words); + + for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { + let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); + if let Some(ref attributes_to_highlight) = self.attributes_to_highlight { + highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + } + documents.push(object); + } + + Ok(SearchResult { + hits: documents, + nb_hits, + query: self.q.clone().unwrap_or_default(), + limit, + offset: self.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + }) + } } #[derive(Serialize)] @@ -105,45 +162,9 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { impl Data { pub fn search>(&self, index: S, search_query: SearchQuery) -> anyhow::Result { - let start = Instant::now(); - let index = self.indexes - .get(&index)? - .ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?; - - let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(&search_query)?; - - let fields_ids_map = index.fields_ids_map()?; - - let displayed_fields = match index.displayed_fields_ids()? { - Some(fields) => fields, - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let attributes_to_highlight = match search_query.attributes_to_highlight { - Some(fields) => fields.iter().map(ToOwned::to_owned).collect(), - None => HashSet::new(), - }; - - let stop_words = fst::Set::default(); - let highlighter = Highlighter::new(&stop_words); - let mut documents = Vec::new(); - for (_id, obkv) in index.documents(&documents_ids)? { - let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); - highlighter.highlight_record(&mut object, &found_words, &attributes_to_highlight); - documents.push(object); + match self.index_controller.index(&index)? { + Some(index) => Ok(search_query.perform(index)?), + None => bail!("index {:?} doesn't exists", index.as_ref()), } - - let processing_time_ms = start.elapsed().as_millis(); - - let result = SearchResult { - hits: documents, - nb_hits, - query: search_query.q.unwrap_or_default(), - offset: search_query.offset.unwrap_or(0), - limit, - processing_time_ms, - }; - - Ok(result) } } diff --git a/src/data/updates.rs b/src/data/updates.rs index 507223d41..d05617361 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,15 +1,12 @@ use std::ops::Deref; use milli::update::{IndexDocumentsMethod, UpdateFormat}; -//use milli::update_store::UpdateStatus; use async_compression::tokio_02::write::GzipEncoder; use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; use super::Data; -use crate::index_controller::IndexController; -use crate::index_controller::{UpdateStatusResponse, Settings}; - +use crate::index_controller::{IndexController, UpdateStatusResponse, Settings}; impl Data { pub async fn add_documents( @@ -39,8 +36,8 @@ impl Data { let file = file.into_std().await; let mmap = unsafe { memmap::Mmap::map(&file)? }; - let indexes = self.indexes.clone(); - let update = tokio::task::spawn_blocking(move ||indexes.add_documents(index, method, format, &mmap[..])).await??; + let index_controller = self.index_controller.clone(); + let update = tokio::task::spawn_blocking(move ||index_controller.add_documents(index, method, format, &mmap[..])).await??; Ok(update.into()) } @@ -49,7 +46,7 @@ impl Data { index: S, settings: Settings ) -> anyhow::Result { - let indexes = self.indexes.clone(); + let indexes = self.index_controller.clone(); let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??; Ok(update.into()) } diff --git a/src/index_controller/_update_store/mod.rs b/src/index_controller/_update_store/mod.rs new file mode 100644 index 000000000..ef8711ded --- /dev/null +++ b/src/index_controller/_update_store/mod.rs @@ -0,0 +1,17 @@ +use std::sync::Arc; + +use heed::Env; + +use super::IndexStore; + +pub struct UpdateStore { + env: Env, + index_store: Arc, +} + +impl UpdateStore { + pub fn new(env: Env, index_store: Arc) -> anyhow::Result { + Ok(Self { env, index_store }) + } +} + diff --git a/src/index_controller/index_store.rs b/src/index_controller/index_store.rs index 6ba6621c3..6652086f9 100644 --- a/src/index_controller/index_store.rs +++ b/src/index_controller/index_store.rs @@ -1,78 +1,12 @@ -use std::fs::File; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; use std::sync::Arc; use std::collections::HashMap; use anyhow::Result; -use chrono::{DateTime, Utc}; -use dashmap::DashMap; -use heed::types::{Str, SerdeBincode}; -use heed::{EnvOpenOptions, Env, Database}; use milli::{Index, FieldsIdsMap, SearchResult, FieldId, facet::FacetType}; -use serde::{Serialize, Deserialize}; use ouroboros::self_referencing; use crate::data::SearchQuery; -const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; -const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; -const INDEXES_DB_NAME: &str = "indexes_db"; - - -#[derive(Debug, Serialize, Deserialize)] -struct IndexStoreMeta { - open_options: EnvOpenOptions, - created_at: DateTime, -} - -impl IndexStoreMeta { - fn from_path(path: impl AsRef) -> Result> { - let mut path = path.as_ref().to_path_buf(); - path.push(CONTROLLER_META_FILENAME); - if path.exists() { - let mut file = File::open(path)?; - let mut buffer = Vec::new(); - let n = file.read_to_end(&mut buffer)?; - let meta: IndexStoreMeta = serde_json::from_slice(&buffer[..n])?; - Ok(Some(meta)) - } else { - Ok(None) - } - } - - fn to_path(self, path: impl AsRef) -> Result<()> { - let mut path = path.as_ref().to_path_buf(); - path.push(CONTROLLER_META_FILENAME); - if path.exists() { - Err(anyhow::anyhow!("Index controller metadata already exists")) - } else { - let mut file = File::create(path)?; - let json = serde_json::to_vec(&self)?; - file.write_all(&json)?; - Ok(()) - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct IndexMetadata { - created_at: DateTime, - open_options: EnvOpenOptions, - uuid: String, -} - -impl IndexMetadata { - fn open_index(self, path: impl AsRef) -> Result { - // create a path in the form "db_path/indexes/index_id" - let mut path = path.as_ref().to_path_buf(); - path.push("indexes"); - path.push(&self.uuid); - Ok(Index::new(self.open_options, path)?) - } -} - - #[self_referencing] pub struct IndexView { pub index: Arc, @@ -136,120 +70,5 @@ impl IndexView { let index = self.borrow_index(); Ok(index.documents(txn, ids.into_iter().copied())?) } - - //pub async fn add_documents( - //&self, - //method: IndexDocumentsMethod, - //format: UpdateFormat, - //mut stream: impl futures::Stream> + Unpin, - //) -> anyhow::Result - //where - //B: Deref, - //E: std::error::Error + Send + Sync + 'static, - //{ - //let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; - //let file = tokio::fs::File::from_std(file?); - //let mut encoder = GzipEncoder::new(file); - - //while let Some(result) = stream.next().await { - //let bytes = &*result?; - //encoder.write_all(&bytes[..]).await?; - //} - - //encoder.shutdown().await?; - //let mut file = encoder.into_inner(); - //file.sync_all().await?; - //let file = file.into_std().await; - //let mmap = unsafe { memmap::Mmap::map(&file)? }; - - //let meta = UpdateMeta::DocumentsAddition { method, format }; - - //let index = self.index.clone(); - //let queue = self.update_store.clone(); - //let update = tokio::task::spawn_blocking(move || queue.register_update(index, meta, &mmap[..])).await??; - //Ok(update.into()) - //} } -pub struct IndexStore { - path: PathBuf, - env: Env, - indexes_db: Database>, - indexes: DashMap)>, -} - -impl IndexStore { - /// Open the index controller from meta found at path, and create a new one if no meta is - /// found. - pub fn new(path: impl AsRef) -> Result { - // If index controller metadata is present, we return the env, otherwise, we create a new - // metadata from scratch before returning a new env. - let path = path.as_ref().to_path_buf(); - let env = match IndexStoreMeta::from_path(&path)? { - Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, - None => { - let mut open_options = EnvOpenOptions::new(); - open_options.map_size(page_size::get() * 1000); - let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; - let created_at = Utc::now(); - let meta = IndexStoreMeta { open_options: open_options.clone(), created_at }; - meta.to_path(&path)?; - env - } - }; - let indexes = DashMap::new(); - let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { - Some(indexes_db) => indexes_db, - None => env.create_database(Some(INDEXES_DB_NAME))?, - }; - - Ok(Self { env, indexes, indexes_db, path }) - } - - pub fn get_or_create>(&self, _name: S) -> Result { - todo!() - } - - /// Get an index with read access to the db. The index are lazily loaded, meaning that we first - /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check - /// for metadata to launch the index. - pub fn get>(&self, name: S) -> Result> { - match self.indexes.get(name.as_ref()) { - Some(entry) => { - let index = entry.1.clone(); - let uuid = entry.0.clone(); - let view = IndexView::try_new(index, |index| index.read_txn(), uuid)?; - Ok(Some(view)) - } - None => { - let txn = self.env.read_txn()?; - match self.indexes_db.get(&txn, name.as_ref())? { - Some(meta) => { - let uuid = meta.uuid.clone(); - let index = Arc::new(meta.open_index(&self.path)?); - self.indexes.insert(name.as_ref().to_owned(), (uuid.clone(), index.clone())); - let view = IndexView::try_new(index, |index| index.read_txn(), uuid)?; - Ok(Some(view)) - } - None => Ok(None) - } - } - } - } - - pub fn get_mut>(&self, _name: S) -> Result> { - todo!() - } - - pub async fn delete_index>(&self, _name:S) -> Result<()> { - todo!() - } - - pub async fn list_indices(&self) -> Result> { - todo!() - } - - pub async fn rename_index(&self, _old: &str, _new: &str) -> Result<()> { - todo!() - } -} diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs new file mode 100644 index 000000000..bfe63459a --- /dev/null +++ b/src/index_controller/local_index_controller/index_store.rs @@ -0,0 +1,188 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use dashmap::DashMap; +use dashmap::mapref::entry::Entry; +use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; +use milli::Index; +use rayon::ThreadPool; +use uuid::Uuid; +use serde::{Serialize, Deserialize}; + +use super::update_store::UpdateStore; +use super::update_handler::UpdateHandler; +use crate::option::IndexerOpts; + +#[derive(Serialize, Deserialize, Debug)] +struct IndexMeta { + update_size: usize, + index_size: usize, + uid: Uuid, +} + +impl IndexMeta { + fn open( + &self, + path: impl AsRef, + thread_pool: Arc, + opt: &IndexerOpts, + ) -> anyhow::Result<(Arc, Arc)> { + let update_path = make_update_db_path(&path, &self.uid); + let index_path = make_index_db_path(&path, &self.uid); + + let mut options = EnvOpenOptions::new(); + options.map_size(self.index_size); + let index = Arc::new(Index::new(options, index_path)?); + + let mut options = EnvOpenOptions::new(); + options.map_size(self.update_size); + let handler = UpdateHandler::new(opt, index.clone(), thread_pool)?; + let update_store = UpdateStore::open(options, update_path, handler)?; + Ok((index, update_store)) + } +} + +pub struct IndexStore { + env: Env, + name_to_uid: DashMap, + name_to_uid_db: Database, + uid_to_index: DashMap, Arc)>, + uid_to_index_db: Database>, + + thread_pool: Arc, + opt: IndexerOpts, +} + +impl IndexStore { + pub fn new(path: impl AsRef, opt: IndexerOpts) -> anyhow::Result { + let env = EnvOpenOptions::new() + .map_size(4096 * 100) + .max_dbs(2) + .open(path)?; + + let name_to_uid = DashMap::new(); + let uid_to_index = DashMap::new(); + let name_to_uid_db = open_or_create_database(&env, Some("name_to_uid"))?; + let uid_to_index_db = open_or_create_database(&env, Some("uid_to_index_db"))?; + + let thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(opt.indexing_jobs.unwrap_or(0)) + .build()?; + let thread_pool = Arc::new(thread_pool); + + Ok(Self { + env, + name_to_uid, + name_to_uid_db, + uid_to_index, + uid_to_index_db, + + thread_pool, + opt, + }) + } + + fn index_uid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { + match self.name_to_uid.entry(name.as_ref().to_string()) { + Entry::Vacant(entry) => { + match self.name_to_uid_db.get(txn, name.as_ref())? { + Some(bytes) => { + let uuid = Uuid::from_slice(bytes)?; + entry.insert(uuid); + Ok(Some(uuid)) + } + None => Ok(None) + } + } + Entry::Occupied(entry) => Ok(Some(entry.get().clone())), + } + } + + fn retrieve_index(&self, txn: &RoTxn, uid: Uuid) -> anyhow::Result, Arc)>> { + match self.uid_to_index.entry(uid.clone()) { + Entry::Vacant(entry) => { + match self.uid_to_index_db.get(txn, uid.as_bytes())? { + Some(meta) => { + let path = self.env.path(); + let (index, updates) = meta.open(path, self.thread_pool.clone(), &self.opt)?; + entry.insert((index.clone(), updates.clone())); + Ok(Some((index, updates))) + }, + None => Ok(None) + } + } + Entry::Occupied(entry) => { + let (index, updates) = entry.get(); + Ok(Some((index.clone(), updates.clone()))) + } + } + } + + fn _get_index(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { + match self.index_uid(&txn, name)? { + Some(uid) => self.retrieve_index(&txn, uid), + None => Ok(None), + } + } + + pub fn index(&self, name: impl AsRef) -> anyhow::Result, Arc)>> { + let txn = self.env.read_txn()?; + self._get_index(&txn, name) + } + + pub fn get_or_create_index( + &self, name: impl AsRef, + update_size: usize, + index_size: usize, + ) -> anyhow::Result<(Arc, Arc)> { + let mut txn = self.env.write_txn()?; + match self._get_index(&txn, name.as_ref())? { + Some(res) => Ok(res), + None => { + let uid = Uuid::new_v4(); + // TODO: clean in case of error + Ok(self.create_index(&mut txn, uid, name, update_size, index_size)?) + }, + } + } + + fn create_index( &self, + txn: &mut RwTxn, + uid: Uuid, + name: impl AsRef, + update_size: usize, + index_size: usize, + ) -> anyhow::Result<(Arc, Arc)> { + let meta = IndexMeta { update_size, index_size, uid: uid.clone() }; + + self.name_to_uid_db.put(txn, name.as_ref(), uid.as_bytes())?; + self.uid_to_index_db.put(txn, uid.as_bytes(), &meta)?; + + let path = self.env.path(); + let (index, update_store) = meta.open(path, self.thread_pool.clone(), &self.opt)?; + + self.name_to_uid.insert(name.as_ref().to_string(), uid); + self.uid_to_index.insert(uid, (index.clone(), update_store.clone())); + + Ok((index, update_store)) + } +} + +fn open_or_create_database(env: &Env, name: Option<&str>) -> anyhow::Result> { + match env.open_database::(name)? { + Some(db) => Ok(db), + None => Ok(env.create_database::(name)?), + } +} + +fn make_update_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { + let mut path = path.as_ref().to_path_buf(); + path.push(format!("update{}", uid)); + path +} + +fn make_index_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { + let mut path = path.as_ref().to_path_buf(); + path.push(format!("index{}", uid)); + path +} diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs new file mode 100644 index 000000000..dc26d0da8 --- /dev/null +++ b/src/index_controller/local_index_controller/mod.rs @@ -0,0 +1,57 @@ +mod update_store; +mod index_store; +mod update_handler; + +use index_store::IndexStore; + +use std::path::Path; +use std::sync::Arc; + +use milli::Index; + +use crate::option::IndexerOpts; +use super::IndexController; + +pub struct LocalIndexController { + indexes: IndexStore, +} + +impl LocalIndexController { + pub fn new(path: impl AsRef, opt: IndexerOpts) -> anyhow::Result { + let indexes = IndexStore::new(path, opt)?; + Ok(Self { indexes }) + } +} + +impl IndexController for LocalIndexController { + fn add_documents>( + &self, + _index: S, + _method: milli::update::IndexDocumentsMethod, + _format: milli::update::UpdateFormat, + _data: &[u8], + ) -> anyhow::Result { + todo!() + } + + fn update_settings>(&self, _index_uid: S, _settings: super::Settings) -> anyhow::Result { + todo!() + } + + fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { + todo!() + } + + fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { + todo!() + } + + fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { + todo!() + } + + fn index(&self, name: impl AsRef) -> anyhow::Result>> { + let index = self.indexes.index(name)?.map(|(i, _)| i); + Ok(index) + } +} diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs new file mode 100644 index 000000000..fae3ad0ae --- /dev/null +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -0,0 +1,206 @@ +use std::io; +use std::sync::Arc; +use std::collections::HashMap; + +use anyhow::Result; +use flate2::read::GzDecoder; +use grenad::CompressionType; +use log::info; +use milli::Index; +use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod}; +use rayon::ThreadPool; + +use crate::index_controller::updates::{Processing, Processed, Failed}; +use crate::index_controller::{UpdateResult, UpdateMeta, Settings, Facets}; +use crate::option::IndexerOpts; +use super::update_store::HandleUpdate; + +pub struct UpdateHandler { + index: Arc, + max_nb_chunks: Option, + chunk_compression_level: Option, + thread_pool: Arc, + log_frequency: usize, + max_memory: usize, + linked_hash_map_size: usize, + chunk_compression_type: CompressionType, + chunk_fusing_shrink_size: u64, +} + +impl UpdateHandler { + pub fn new( + opt: &IndexerOpts, + index: Arc, + thread_pool: Arc, + ) -> anyhow::Result { + Ok(Self { + index, + max_nb_chunks: opt.max_nb_chunks, + chunk_compression_level: opt.chunk_compression_level, + thread_pool, + log_frequency: opt.log_every_n, + max_memory: opt.max_memory.get_bytes() as usize, + linked_hash_map_size: opt.linked_hash_map_size, + chunk_compression_type: opt.chunk_compression_type, + chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), + }) + } + + fn update_buidler(&self, update_id: u64) -> UpdateBuilder { + // We prepare the update by using the update builder. + let mut update_builder = UpdateBuilder::new(update_id); + if let Some(max_nb_chunks) = self.max_nb_chunks { + update_builder.max_nb_chunks(max_nb_chunks); + } + if let Some(chunk_compression_level) = self.chunk_compression_level { + update_builder.chunk_compression_level(chunk_compression_level); + } + update_builder.thread_pool(&self.thread_pool); + update_builder.log_every_n(self.log_frequency); + update_builder.max_memory(self.max_memory); + update_builder.linked_hash_map_size(self.linked_hash_map_size); + update_builder.chunk_compression_type(self.chunk_compression_type); + update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); + update_builder + } + + fn update_documents( + &self, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: &[u8], + update_builder: UpdateBuilder, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.index.write_txn()?; + let mut builder = update_builder.index_documents(&mut wtxn, &self.index); + builder.update_format(format); + builder.index_documents_method(method); + + let gzipped = true; + let reader = if gzipped { + Box::new(GzDecoder::new(content)) + } else { + Box::new(content) as Box + }; + + let result = builder.execute(reader, |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(addition_result) => wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.index.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, &self.index); + + match builder.execute() { + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn update_settings(&self, settings: &Settings, update_builder: UpdateBuilder) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.index.write_txn()?; + let mut builder = update_builder.settings(&mut wtxn, &self.index); + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names.clone()), + None => builder.reset_searchable_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names.clone()), + None => builder.reset_displayed_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref facet_types) = settings.faceted_attributes { + let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + builder.set_faceted_fields(facet_types); + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref criteria) = settings.criteria { + match criteria { + Some(criteria) => builder.set_criteria(criteria.clone()), + None => builder.reset_criteria(), + } + } + + let result = builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn update_facets( + &self, + levels: &Facets, + update_builder: UpdateBuilder + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.index.write_txn()?; + let mut builder = update_builder.facets(&mut wtxn, &self.index); + if let Some(value) = levels.level_group_size { + builder.level_group_size(value); + } + if let Some(value) = levels.min_level_size { + builder.min_level_size(value); + } + match builder.execute() { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } +} + +impl HandleUpdate for UpdateHandler { + fn handle_update( + &mut self, + update_id: u64, + meta: Processing, + content: &[u8] + ) -> Result, Failed> { + use UpdateMeta::*; + + let update_builder = self.update_buidler(update_id); + + let result = match meta.meta() { + DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), + ClearDocuments => self.clear_documents(update_builder), + Settings(settings) => self.update_settings(settings, update_builder), + Facets(levels) => self.update_facets(levels, update_builder), + }; + + match result { + Ok(result) => Ok(meta.process(result)), + Err(e) => Err(meta.fail(e.to_string())), + } + } +} + diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs new file mode 100644 index 000000000..94543b0a1 --- /dev/null +++ b/src/index_controller/local_index_controller/update_store.rs @@ -0,0 +1,311 @@ +use std::path::Path; +use std::sync::{Arc, RwLock}; + +use crossbeam_channel::Sender; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::{EnvOpenOptions, Env, Database}; + +use crate::index_controller::updates::*; +use crate::index_controller::{UpdateMeta, UpdateResult}; + +type BEU64 = heed::zerocopy::U64; + +#[derive(Clone)] +pub struct UpdateStore { + env: Env, + pending_meta: Database, SerdeJson>>, + pending: Database, ByteSlice>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, + notification_sender: Sender<()>, +} + +pub trait HandleUpdate { + fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed>; +} + +impl UpdateStore { + pub fn open( + mut options: EnvOpenOptions, + path: P, + mut update_handler: U, + ) -> heed::Result> + where + P: AsRef, + U: HandleUpdate + Send + 'static, + { + options.max_dbs(5); + + let env = options.open(path)?; + let pending_meta = env.create_database(Some("pending-meta"))?; + let pending = env.create_database(Some("pending"))?; + let processed_meta = env.create_database(Some("processed-meta"))?; + let aborted_meta = env.create_database(Some("aborted-meta"))?; + let failed_meta = env.create_database(Some("failed-meta"))?; + let processing = Arc::new(RwLock::new(None)); + + let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + // Send a first notification to trigger the process. + let _ = notification_sender.send(()); + + let update_store = Arc::new(UpdateStore { + env, + pending, + pending_meta, + processed_meta, + aborted_meta, + notification_sender, + failed_meta, + processing, + }); + + let update_store_cloned = update_store.clone(); + std::thread::spawn(move || { + // Block and wait for something to process. + for () in notification_receiver { + loop { + match update_store_cloned.process_pending_update(&mut update_handler) { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => eprintln!("error while processing update: {}", e), + } + } + } + }); + + Ok(update_store) + } + + /// Returns the new biggest id to use to store the new update. + fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { + let last_pending = self.pending_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_processed = self.processed_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_aborted = self.aborted_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_update_id = [last_pending, last_processed, last_aborted] + .iter() + .copied() + .flatten() + .max(); + + match last_update_id { + Some(last_id) => Ok(last_id + 1), + None => Ok(0), + } + } + + /// Registers the update content in the pending store and the meta + /// into the pending-meta store. Returns the new unique update id. + pub fn register_update( + &self, + meta: UpdateMeta, + content: &[u8] + ) -> heed::Result> { + let mut wtxn = self.env.write_txn()?; + + // We ask the update store to give us a new update id, this is safe, + // no other update can have the same id because we use a write txn before + // asking for the id and registering it so other update registering + // will be forced to wait for a new write txn. + let update_id = self.new_update_id(&wtxn)?; + let update_key = BEU64::new(update_id); + + let meta = Pending::new(meta, update_id); + self.pending_meta.put(&mut wtxn, &update_key, &meta)?; + self.pending.put(&mut wtxn, &update_key, content)?; + + wtxn.commit()?; + + if let Err(e) = self.notification_sender.try_send(()) { + assert!(!e.is_disconnected(), "update notification channel is disconnected"); + } + Ok(meta) + } + /// Executes the user provided function on the next pending update (the one with the lowest id). + /// This is asynchronous as it let the user process the update with a read-only txn and + /// only writing the result meta to the processed-meta store *after* it has been processed. + fn process_pending_update(&self, handler: &mut U) -> heed::Result> + where + U: HandleUpdate + Send + 'static, + { + // Create a read transaction to be able to retrieve the pending update in order. + let rtxn = self.env.read_txn()?; + let first_meta = self.pending_meta.first(&rtxn)?; + + // If there is a pending update we process and only keep + // a reader while processing it, not a writer. + match first_meta { + Some((first_id, pending)) => { + let first_content = self.pending + .get(&rtxn, &first_id)? + .expect("associated update content"); + + // we cahnge the state of the update from pending to processing before we pass it + // to the update handler. Processing store is non persistent to be able recover + // from a failure + let processing = pending.processing(); + self.processing + .write() + .unwrap() + .replace(processing.clone()); + // Process the pending update using the provided user function. + let result = handler.handle_update(first_id.get(), processing, first_content); + drop(rtxn); + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.processing + .write() + .unwrap() + .take(); + self.pending_meta.delete(&mut wtxn, &first_id)?; + self.pending.delete(&mut wtxn, &first_id)?; + match result { + Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, + Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, + } + wtxn.commit()?; + + Ok(Some(())) + }, + None => Ok(None) + } + } + + /// The id and metadata of the update that is currently being processed, + /// `None` if no update is being processed. + pub fn processing_update(&self) -> heed::Result)>> { + let rtxn = self.env.read_txn()?; + match self.pending_meta.first(&rtxn)? { + Some((key, meta)) => Ok(Some((key.get(), meta))), + None => Ok(None), + } + } + + /// Execute the user defined function with the meta-store iterators, the first + /// iterator is the *processed* meta one, the second the *aborted* meta one + /// and, the last is the *pending* meta one. + pub fn iter_metas(&self, mut f: F) -> heed::Result + where + F: for<'a> FnMut( + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + ) -> heed::Result, + { + let rtxn = self.env.read_txn()?; + + // We get the pending, processed and aborted meta iterators. + let processed_iter = self.processed_meta.iter(&rtxn)?; + let aborted_iter = self.aborted_meta.iter(&rtxn)?; + let pending_iter = self.pending_meta.iter(&rtxn)?; + let processing = self.processing.read().unwrap().clone(); + let failed_iter = self.failed_meta.iter(&rtxn)?; + + // We execute the user defined function with both iterators. + (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + } + + /// Returns the update associated meta or `None` if the update doesn't exist. + pub fn meta(&self, update_id: u64) -> heed::Result>> { + let rtxn = self.env.read_txn()?; + let key = BEU64::new(update_id); + + if let Some(ref meta) = *self.processing.read().unwrap() { + if meta.id() == update_id { + return Ok(Some(UpdateStatus::Processing(meta.clone()))); + } + } + + if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Pending(meta))); + } + + if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Processed(meta))); + } + + if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Aborted(meta))); + } + + if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Failed(meta))); + } + + Ok(None) + } + + /// Aborts an update, an aborted update content is deleted and + /// the meta of it is moved into the aborted updates database. + /// + /// Trying to abort an update that is currently being processed, an update + /// that as already been processed or which doesn't actually exist, will + /// return `None`. + pub fn abort_update(&self, update_id: u64) -> heed::Result>> { + let mut wtxn = self.env.write_txn()?; + let key = BEU64::new(update_id); + + // We cannot abort an update that is currently being processed. + if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { + return Ok(None); + } + + let pending = match self.pending_meta.get(&wtxn, &key)? { + Some(meta) => meta, + None => return Ok(None), + }; + + let aborted = pending.abort(); + + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + + wtxn.commit()?; + + Ok(Some(aborted)) + } + + /// Aborts all the pending updates, and not the one being currently processed. + /// Returns the update metas and ids that were successfully aborted. + pub fn abort_pendings(&self) -> heed::Result)>> { + let mut wtxn = self.env.write_txn()?; + let mut aborted_updates = Vec::new(); + + // We skip the first pending update as it is currently being processed. + for result in self.pending_meta.iter(&wtxn)?.skip(1) { + let (key, pending) = result?; + let id = key.get(); + aborted_updates.push((id, pending.abort())); + } + + for (id, aborted) in &aborted_updates { + let key = BEU64::new(*id); + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + } + + wtxn.commit()?; + + Ok(aborted_updates) + } +} diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index c927a6c5b..d59575f7d 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,18 +1,19 @@ -mod index_store; -mod update_store; +mod local_index_controller; +mod updates; -pub use index_store::IndexStore; -pub use update_store::UpdateStore; +pub use local_index_controller::LocalIndexController; -use std::num::NonZeroUsize; -use std::ops::Deref; use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::sync::Arc; use anyhow::Result; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use milli::update_store::{Processed, Processing, Failed, Pending, Aborted}; +use milli::Index; +use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; +use updates::{Processed, Processing, Failed, Pending, Aborted}; + pub type UpdateStatusResponse = UpdateStatus; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -89,7 +90,7 @@ impl Settings { } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { - //DocumentsAddition(DocumentAdditionResult), + DocumentsAddition(DocumentAdditionResult), Other, } @@ -97,7 +98,7 @@ pub enum UpdateResult { /// for read access which is provided, and write access which must be provided. This allows the /// implementer to define the behaviour of write accesses to the indices, and abstract the /// scheduling of the updates. The implementer must be able to provide an instance of `IndexStore` -pub trait IndexController: Deref { +pub trait IndexController { /* * Write operations @@ -141,5 +142,7 @@ pub trait IndexController: Deref { ) -> Result, Failed> { todo!() } -} + /// Returns, if it exists, an `IndexView` to the requested index. + fn index(&self, uid: impl AsRef) -> anyhow::Result>>; +} diff --git a/src/index_controller/update_store/mod.rs b/src/index_controller/update_store/mod.rs deleted file mode 100644 index 84db2f63d..000000000 --- a/src/index_controller/update_store/mod.rs +++ /dev/null @@ -1,49 +0,0 @@ -use std::ops::Deref; - -use super::{IndexStore, IndexController}; - -pub struct UpdateStore { - index_store: IndexStore, -} - -impl Deref for UpdateStore { - type Target = IndexStore; - - fn deref(&self) -> &Self::Target { - &self.index_store - } -} - -impl UpdateStore { - pub fn new(index_store: IndexStore) -> Self { - Self { index_store } - } -} - -impl IndexController for UpdateStore { - fn add_documents>( - &self, - _index: S, - _method: milli::update::IndexDocumentsMethod, - _format: milli::update::UpdateFormat, - _data: &[u8], - ) -> anyhow::Result { - todo!() - } - - fn update_settings>(&self, _index_uid: S, _settings: crate::index_controller::Settings) -> anyhow::Result { - todo!() - } - - fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { - todo!() - } - - fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { - todo!() - } - - fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { - todo!() - } -} diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs new file mode 100644 index 000000000..7c67ea8c2 --- /dev/null +++ b/src/index_controller/updates.rs @@ -0,0 +1,167 @@ +use chrono::{Utc, DateTime}; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Pending { + update_id: u64, + meta: M, + enqueued_at: DateTime, +} + +impl Pending { + pub fn new(meta: M, update_id: u64) -> Self { + Self { + enqueued_at: Utc::now(), + meta, + update_id, + } + } + + pub fn processing(self) -> Processing { + Processing { + from: self, + started_processing_at: Utc::now(), + } + } + + pub fn abort(self) -> Aborted { + Aborted { + from: self, + aborted_at: Utc::now(), + } + } + + pub fn meta(&self) -> &M { + &self.meta + } + + pub fn id(&self) -> u64 { + self.update_id + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Processed { + success: N, + processed_at: DateTime, + #[serde(flatten)] + from: Processing, +} + +impl Processed { + pub fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Processing { + #[serde(flatten)] + from: Pending, + started_processing_at: DateTime, +} + +impl Processing { + pub fn id(&self) -> u64 { + self.from.id() + } + + pub fn meta(&self) -> &M { + self.from.meta() + } + + pub fn process(self, meta: N) -> Processed { + Processed { + success: meta, + from: self, + processed_at: Utc::now(), + } + } + + pub fn fail(self, error: E) -> Failed { + Failed { + from: self, + error, + failed_at: Utc::now(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Aborted { + #[serde(flatten)] + from: Pending, + aborted_at: DateTime, +} + +impl Aborted { + pub fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Failed { + #[serde(flatten)] + from: Processing, + error: E, + failed_at: DateTime, +} + +impl Failed { + pub fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize)] +#[serde(tag = "status")] +pub enum UpdateStatus { + Processing(Processing), + Pending(Pending), + Processed(Processed), + Aborted(Aborted), + Failed(Failed), +} + +impl UpdateStatus { + pub fn id(&self) -> u64 { + match self { + UpdateStatus::Processing(u) => u.id(), + UpdateStatus::Pending(u) => u.id(), + UpdateStatus::Processed(u) => u.id(), + UpdateStatus::Aborted(u) => u.id(), + UpdateStatus::Failed(u) => u.id(), + } + } +} + +impl From> for UpdateStatus { + fn from(other: Pending) -> Self { + Self::Pending(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Aborted) -> Self { + Self::Aborted(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Processed) -> Self { + Self::Processed(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Processing) -> Self { + Self::Processing(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Failed) -> Self { + Self::Failed(other) + } +} From 81832028689de6c298c38acffa03fbab8e45afd4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 15:14:48 +0100 Subject: [PATCH 021/522] documetn addition and search --- src/data/mod.rs | 7 ++- src/data/updates.rs | 7 +-- .../local_index_controller/index_store.rs | 45 +++++++++++++++---- .../local_index_controller/mod.rs | 34 ++++++++++---- .../local_index_controller/update_handler.rs | 4 ++ src/index_controller/mod.rs | 18 ++------ 6 files changed, 79 insertions(+), 36 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 7494792bc..175aedba5 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -61,7 +61,12 @@ impl Data { let path = options.db_path.clone(); let indexer_opts = options.indexer_options.clone(); create_dir_all(&path)?; - let index_controller = LocalIndexController::new(&path, indexer_opts)?; + let index_controller = LocalIndexController::new( + &path, + indexer_opts, + options.max_mdb_size.get_bytes(), + options.max_udb_size.get_bytes(), + )?; let indexes = Arc::new(index_controller); let mut api_keys = ApiKeys { diff --git a/src/data/updates.rs b/src/data/updates.rs index d05617361..880579bf6 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -6,7 +6,8 @@ use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; use super::Data; -use crate::index_controller::{IndexController, UpdateStatusResponse, Settings}; +use crate::index_controller::{IndexController, Settings, UpdateResult, UpdateMeta}; +use crate::index_controller::updates::UpdateStatus; impl Data { pub async fn add_documents( @@ -15,7 +16,7 @@ impl Data { method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, - ) -> anyhow::Result + ) -> anyhow::Result> where B: Deref, E: std::error::Error + Send + Sync + 'static, @@ -45,7 +46,7 @@ impl Data { &self, index: S, settings: Settings - ) -> anyhow::Result { + ) -> anyhow::Result> { let indexes = self.index_controller.clone(); let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??; Ok(update.into()) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index bfe63459a..4c9a98472 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -1,4 +1,5 @@ use std::path::{Path, PathBuf}; +use std::fs::create_dir_all; use std::sync::Arc; use dashmap::DashMap; @@ -8,6 +9,7 @@ use milli::Index; use rayon::ThreadPool; use uuid::Uuid; use serde::{Serialize, Deserialize}; +use log::warn; use super::update_store::UpdateStore; use super::update_handler::UpdateHandler; @@ -15,8 +17,8 @@ use crate::option::IndexerOpts; #[derive(Serialize, Deserialize, Debug)] struct IndexMeta { - update_size: usize, - index_size: usize, + update_size: u64, + index_size: u64, uid: Uuid, } @@ -30,12 +32,15 @@ impl IndexMeta { let update_path = make_update_db_path(&path, &self.uid); let index_path = make_index_db_path(&path, &self.uid); + create_dir_all(&update_path)?; + create_dir_all(&index_path)?; + let mut options = EnvOpenOptions::new(); - options.map_size(self.index_size); + options.map_size(self.index_size as usize); let index = Arc::new(Index::new(options, index_path)?); let mut options = EnvOpenOptions::new(); - options.map_size(self.update_size); + options.map_size(self.update_size as usize); let handler = UpdateHandler::new(opt, index.clone(), thread_pool)?; let update_store = UpdateStore::open(options, update_path, handler)?; Ok((index, update_store)) @@ -132,8 +137,8 @@ impl IndexStore { pub fn get_or_create_index( &self, name: impl AsRef, - update_size: usize, - index_size: usize, + update_size: u64, + index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { let mut txn = self.env.write_txn()?; match self._get_index(&txn, name.as_ref())? { @@ -141,17 +146,39 @@ impl IndexStore { None => { let uid = Uuid::new_v4(); // TODO: clean in case of error - Ok(self.create_index(&mut txn, uid, name, update_size, index_size)?) + let result = self.create_index(&mut txn, uid, name, update_size, index_size); + match result { + Ok((index, update_store)) => { + match txn.commit() { + Ok(_) => Ok((index, update_store)), + Err(e) => { + self.clean_uid(&uid); + Err(anyhow::anyhow!("error creating index: {}", e)) + } + } + } + Err(e) => { + self.clean_uid(&uid); + Err(e) + } + } }, } } + /// removes all data acociated with an index Uuid. This is called when index creation failed + /// and outstanding files and data need to be cleaned. + fn clean_uid(&self, _uid: &Uuid) { + // TODO! + warn!("creating cleanup is not yet implemented"); + } + fn create_index( &self, txn: &mut RwTxn, uid: Uuid, name: impl AsRef, - update_size: usize, - index_size: usize, + update_size: u64, + index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { let meta = IndexMeta { update_size, index_size, uid: uid.clone() }; diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index dc26d0da8..00a6cc363 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -11,30 +11,46 @@ use milli::Index; use crate::option::IndexerOpts; use super::IndexController; +use super::updates::UpdateStatus; +use super::{UpdateMeta, UpdateResult}; pub struct LocalIndexController { indexes: IndexStore, + update_db_size: u64, + index_db_size: u64, } impl LocalIndexController { - pub fn new(path: impl AsRef, opt: IndexerOpts) -> anyhow::Result { + pub fn new( + path: impl AsRef, + opt: IndexerOpts, + index_db_size: u64, + update_db_size: u64, + ) -> anyhow::Result { let indexes = IndexStore::new(path, opt)?; - Ok(Self { indexes }) + Ok(Self { indexes, index_db_size, update_db_size }) } } impl IndexController for LocalIndexController { fn add_documents>( &self, - _index: S, - _method: milli::update::IndexDocumentsMethod, - _format: milli::update::UpdateFormat, - _data: &[u8], - ) -> anyhow::Result { - todo!() + index: S, + method: milli::update::IndexDocumentsMethod, + format: milli::update::UpdateFormat, + data: &[u8], + ) -> anyhow::Result> { + let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; + let meta = UpdateMeta::DocumentsAddition { method, format }; + let pending = update_store.register_update(meta, data).unwrap(); + Ok(pending.into()) } - fn update_settings>(&self, _index_uid: S, _settings: super::Settings) -> anyhow::Result { + fn update_settings>( + &self, + _index_uid: S, + _settings: super::Settings + ) -> anyhow::Result> { todo!() } diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index fae3ad0ae..24b9ab405 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -188,6 +188,8 @@ impl HandleUpdate for UpdateHandler { ) -> Result, Failed> { use UpdateMeta::*; + println!("handling update {}", update_id); + let update_builder = self.update_buidler(update_id); let result = match meta.meta() { @@ -197,6 +199,8 @@ impl HandleUpdate for UpdateHandler { Facets(levels) => self.update_facets(levels, update_builder), }; + println!("{:?}", result); + match result { Ok(result) => Ok(meta.process(result)), Err(e) => Err(meta.fail(e.to_string())), diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index d59575f7d..0907ba0c8 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,5 +1,5 @@ mod local_index_controller; -mod updates; +pub mod updates; pub use local_index_controller::LocalIndexController; @@ -12,9 +12,8 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; -use updates::{Processed, Processing, Failed, Pending, Aborted}; +use updates::{Processed, Processing, Failed, UpdateStatus}; -pub type UpdateStatusResponse = UpdateStatus; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] @@ -33,15 +32,6 @@ pub struct Facets { pub min_level_size: Option, } -#[derive(Debug, Clone, Serialize)] -#[serde(tag = "type")] -pub enum UpdateStatus { - Pending { update_id: u64, meta: Pending }, - Progressing { update_id: u64, meta: P }, - Processed { update_id: u64, meta: Processed }, - Aborted { update_id: u64, meta: Aborted }, -} - fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> where T: Deserialize<'de>, D: Deserializer<'de> @@ -116,11 +106,11 @@ pub trait IndexController { method: IndexDocumentsMethod, format: UpdateFormat, data: &[u8], - ) -> anyhow::Result; + ) -> anyhow::Result>; /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. - fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; + fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result>; /// Create an index with the given `index_uid`. fn create_index>(&self, index_uid: S) -> Result<()>; From 4119ae865528dfad19f935fea300c5f9866c18a6 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 16:57:53 +0100 Subject: [PATCH 022/522] setttings update --- src/index_controller/local_index_controller/mod.rs | 9 ++++++--- .../local_index_controller/update_handler.rs | 4 ---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 00a6cc363..f70050512 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -48,10 +48,13 @@ impl IndexController for LocalIndexController { fn update_settings>( &self, - _index_uid: S, - _settings: super::Settings + index: S, + settings: super::Settings ) -> anyhow::Result> { - todo!() + let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; + let meta = UpdateMeta::Settings(settings); + let pending = update_store.register_update(meta, &[]).unwrap(); + Ok(pending.into()) } fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index 24b9ab405..fae3ad0ae 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -188,8 +188,6 @@ impl HandleUpdate for UpdateHandler { ) -> Result, Failed> { use UpdateMeta::*; - println!("handling update {}", update_id); - let update_builder = self.update_buidler(update_id); let result = match meta.meta() { @@ -199,8 +197,6 @@ impl HandleUpdate for UpdateHandler { Facets(levels) => self.update_facets(levels, update_builder), }; - println!("{:?}", result); - match result { Ok(result) => Ok(meta.process(result)), Err(e) => Err(meta.fail(e.to_string())), From 60371b9dcf77bcbbb3aea3a27d71f6386b50c2f7 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 17:20:51 +0100 Subject: [PATCH 023/522] get update id --- src/data/updates.rs | 36 ++++++++--------- .../local_index_controller/mod.rs | 8 ++++ src/index_controller/mod.rs | 2 + src/routes/index.rs | 39 +++++++++---------- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 880579bf6..6f44ec57d 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -10,7 +10,7 @@ use crate::index_controller::{IndexController, Settings, UpdateResult, UpdateMet use crate::index_controller::updates::UpdateStatus; impl Data { - pub async fn add_documents( + pub async fn add_documents( &self, index: S, method: IndexDocumentsMethod, @@ -52,24 +52,24 @@ impl Data { Ok(update.into()) } - //#[inline] - //pub fn get_update_status>(&self, _index: S, uid: u64) -> anyhow::Result>> { - //self.indexes.get_update_status(uid) - //} + #[inline] + pub fn get_update_status>(&self, index: S, uid: u64) -> anyhow::Result>> { + self.index_controller.update_status(index, uid) + } //pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { - //let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { - //let mut metas = processing - //.map(UpdateStatus::from) - //.into_iter() - //.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.collect::>(); - //metas.sort_by(|a, b| a.id().cmp(&b.id())); - //Ok(metas) - //})?; - //Ok(result) + //let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { + //let mut metas = processing + //.map(UpdateStatus::from) + //.into_iter() + //.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.collect::>(); + //metas.sort_by(|a, b| a.id().cmp(&b.id())); + //Ok(metas) + //})?; + //Ok(result) //} } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index f70050512..b36009ce5 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -8,6 +8,7 @@ use std::path::Path; use std::sync::Arc; use milli::Index; +use anyhow::bail; use crate::option::IndexerOpts; use super::IndexController; @@ -73,4 +74,11 @@ impl IndexController for LocalIndexController { let index = self.indexes.index(name)?.map(|(i, _)| i); Ok(index) } + + fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>> { + match self.indexes.index(&index)? { + Some((_, update_store)) => Ok(update_store.meta(id)?), + None => bail!("index {:?} doesn't exist", index.as_ref()), + } + } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0907ba0c8..b0d75a266 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -135,4 +135,6 @@ pub trait IndexController { /// Returns, if it exists, an `IndexView` to the requested index. fn index(&self, uid: impl AsRef) -> anyhow::Result>>; + + fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>>; } diff --git a/src/routes/index.rs b/src/routes/index.rs index fa4ae0679..cccf28a2d 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -1,7 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; -//use log::error; +use log::error; use serde::{Deserialize, Serialize}; use crate::Data; @@ -94,8 +94,8 @@ async fn delete_index( #[derive(Deserialize)] struct UpdateParam { - _index_uid: String, - _update_id: u64, + index_uid: String, + update_id: u64, } #[get( @@ -103,24 +103,23 @@ struct UpdateParam { wrap = "Authentication::Private" )] async fn get_update_status( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() - //let result = data.get_update_status(&path.index_uid, path.update_id); - //match result { - //Ok(Some(meta)) => { - //let json = serde_json::to_string(&meta).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Ok(None) => { - //todo!() - //} - //Err(e) => { - //error!("{}", e); - //todo!() - //} - //} + let result = data.get_update_status(&path.index_uid, path.update_id); + match result { + Ok(Some(meta)) => { + let json = serde_json::to_string(&meta).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Ok(None) => { + todo!() + } + Err(e) => { + error!("{}", e); + todo!() + } + } } #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] From 6c63ee6798db69cd19bf82021afd5984ab5c7b0d Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 18:32:24 +0100 Subject: [PATCH 024/522] implement list all indexes --- Cargo.lock | 14 ++++++++-- Cargo.toml | 1 + src/data/updates.rs | 18 +++---------- .../local_index_controller/mod.rs | 22 +++++++++++++++ src/index_controller/mod.rs | 1 + src/routes/index.rs | 27 +++++++++---------- 6 files changed, 52 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ceaef691..fc102336a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1452,6 +1452,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -1650,6 +1659,7 @@ dependencies = [ "heed", "http", "indexmap", + "itertools 0.10.0", "jemallocator", "log", "main_error", @@ -1745,7 +1755,7 @@ dependencies = [ "grenad", "heed", "human_format", - "itertools", + "itertools 0.9.0", "jemallocator", "levenshtein_automata", "linked-hash-map", @@ -3699,6 +3709,6 @@ checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" dependencies = [ "cc", "glob", - "itertools", + "itertools 0.9.0", "libc", ] diff --git a/Cargo.toml b/Cargo.toml index eb2fca43d..f68414beb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,7 @@ page_size = "0.4.2" obkv = "0.1.1" ouroboros = "0.8.0" uuid = "0.8.2" +itertools = "0.10.0" [dependencies.sentry] default-features = false diff --git a/src/data/updates.rs b/src/data/updates.rs index 6f44ec57d..194ec346b 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -57,19 +57,7 @@ impl Data { self.index_controller.update_status(index, uid) } - //pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { - //let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { - //let mut metas = processing - //.map(UpdateStatus::from) - //.into_iter() - //.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.collect::>(); - //metas.sort_by(|a, b| a.id().cmp(&b.id())); - //Ok(metas) - //})?; - //Ok(result) - //} + pub fn get_updates_status(&self, index: &str) -> anyhow::Result>> { + self.index_controller.all_update_status(index) + } } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index b36009ce5..debc15a1a 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use milli::Index; use anyhow::bail; +use itertools::Itertools; use crate::option::IndexerOpts; use super::IndexController; @@ -81,4 +82,25 @@ impl IndexController for LocalIndexController { None => bail!("index {:?} doesn't exist", index.as_ref()), } } + + fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>> { + match self.indexes.index(index)? { + Some((_, update_store)) => { + let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { + Ok(processing + .map(UpdateStatus::from) + .into_iter() + .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) + .chain(aborted.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .chain(processed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .chain(failed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .sorted_by(|a, b| a.id().cmp(&b.id())) + .collect()) + })?; + Ok(updates) + } + None => Ok(Vec::new()) + } + + } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index b0d75a266..f1ba8f7ce 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -137,4 +137,5 @@ pub trait IndexController { fn index(&self, uid: impl AsRef) -> anyhow::Result>>; fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>>; + fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>>; } diff --git a/src/routes/index.rs b/src/routes/index.rs index cccf28a2d..774039f2b 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -124,19 +124,18 @@ async fn get_update_status( #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] async fn get_all_updates_status( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() - //let result = data.get_updates_status(&path.index_uid); - //match result { - //Ok(metas) => { - //let json = serde_json::to_string(&metas).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Err(e) => { - //error!("{}", e); - //todo!() - //} - //} + let result = data.get_updates_status(&path.index_uid); + match result { + Ok(metas) => { + let json = serde_json::to_string(&metas).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + todo!() + } + } } From e9c95f66231d5059f65d8bd13be977611bd593e6 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 19:43:54 +0100 Subject: [PATCH 025/522] remove useless files --- Cargo.lock | 2 +- src/index_controller/_update_store/mod.rs | 17 - src/index_controller/index_store.rs | 74 --- src/updates/mod.rs | 318 ------------ src/updates/settings.rs | 61 --- src/updates/update_store.rs | 581 ---------------------- 6 files changed, 1 insertion(+), 1052 deletions(-) delete mode 100644 src/index_controller/_update_store/mod.rs delete mode 100644 src/index_controller/index_store.rs delete mode 100644 src/updates/mod.rs delete mode 100644 src/updates/settings.rs delete mode 100644 src/updates/update_store.rs diff --git a/Cargo.lock b/Cargo.lock index fc102336a..4471f5127 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1628,7 +1628,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.18.1" +version = "0.18.0" dependencies = [ "actix-http", ] diff --git a/src/index_controller/_update_store/mod.rs b/src/index_controller/_update_store/mod.rs deleted file mode 100644 index ef8711ded..000000000 --- a/src/index_controller/_update_store/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::sync::Arc; - -use heed::Env; - -use super::IndexStore; - -pub struct UpdateStore { - env: Env, - index_store: Arc, -} - -impl UpdateStore { - pub fn new(env: Env, index_store: Arc) -> anyhow::Result { - Ok(Self { env, index_store }) - } -} - diff --git a/src/index_controller/index_store.rs b/src/index_controller/index_store.rs deleted file mode 100644 index 6652086f9..000000000 --- a/src/index_controller/index_store.rs +++ /dev/null @@ -1,74 +0,0 @@ -use std::sync::Arc; -use std::collections::HashMap; - -use anyhow::Result; -use milli::{Index, FieldsIdsMap, SearchResult, FieldId, facet::FacetType}; -use ouroboros::self_referencing; - -use crate::data::SearchQuery; - -#[self_referencing] -pub struct IndexView { - pub index: Arc, - #[borrows(index)] - #[covariant] - pub txn: heed::RoTxn<'this>, - uuid: String, -} - -impl IndexView { - pub fn search(&self, search_query: &SearchQuery) -> Result { - self.with(|this| { - let mut search = this.index.search(&this.txn); - if let Some(query) = &search_query.q { - search.query(query); - } - - if let Some(offset) = search_query.offset { - search.offset(offset); - } - - let limit = search_query.limit; - search.limit(limit); - - Ok(search.execute()?) - }) - } - - #[inline] - pub fn fields_ids_map(&self) -> Result { - self.with(|this| Ok(this.index.fields_ids_map(&this.txn)?)) - - } - - #[inline] - pub fn displayed_fields_ids(&self) -> Result>> { - self.with(|this| Ok(this.index.displayed_fields_ids(&this.txn)?)) - } - - #[inline] - pub fn displayed_fields(&self) -> Result>> { - self.with(|this| Ok(this.index - .displayed_fields(&this.txn)? - .map(|fields| fields.into_iter().map(String::from).collect()))) - } - - #[inline] - pub fn searchable_fields(&self) -> Result>> { - self.with(|this| Ok(this.index - .searchable_fields(&this.txn)? - .map(|fields| fields.into_iter().map(String::from).collect()))) - } - - #[inline] - pub fn faceted_fields(&self) -> Result> { - self.with(|this| Ok(this.index.faceted_fields(&this.txn)?)) - } - - pub fn documents(&self, ids: &[u32]) -> Result)>> { - let txn = self.borrow_txn(); - let index = self.borrow_index(); - Ok(index.documents(txn, ids.into_iter().copied())?) - } -} - diff --git a/src/updates/mod.rs b/src/updates/mod.rs deleted file mode 100644 index 7e0802595..000000000 --- a/src/updates/mod.rs +++ /dev/null @@ -1,318 +0,0 @@ -mod settings; -mod update_store; - -pub use settings::{Settings, Facets}; - -use std::io; -use std::sync::Arc; -use std::fs::create_dir_all; -use std::collections::HashMap; - -use anyhow::Result; -use byte_unit::Byte; -use flate2::read::GzDecoder; -use grenad::CompressionType; -use log::info; -use milli::Index; -use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod, DocumentAdditionResult }; -use milli::update_store::{UpdateStore, UpdateHandler as Handler, UpdateStatus, Processing, Processed, Failed}; -use rayon::ThreadPool; -use serde::{Serialize, Deserialize}; -use structopt::StructOpt; - -use crate::option::Opt; - -pub type UpdateStatusResponse = UpdateStatus; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum UpdateMeta { - DocumentsAddition { method: IndexDocumentsMethod, format: UpdateFormat }, - ClearDocuments, - Settings(Settings), - Facets(Facets), -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum UpdateMetaProgress { - DocumentsAddition { - step: usize, - total_steps: usize, - current: usize, - total: Option, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateResult { - DocumentsAddition(DocumentAdditionResult), - Other, -} - -#[derive(Clone)] -pub struct UpdateQueue { - inner: Arc>, -} - -#[derive(Debug, Clone, StructOpt)] -pub struct IndexerOpts { - /// The amount of documents to skip before printing - /// a log regarding the indexing advancement. - #[structopt(long, default_value = "100000")] // 100k - pub log_every_n: usize, - - /// MTBL max number of chunks in bytes. - #[structopt(long)] - pub max_nb_chunks: Option, - - /// The maximum amount of memory to use for the MTBL buffer. It is recommended - /// to use something like 80%-90% of the available memory. - /// - /// It is automatically split by the number of jobs e.g. if you use 7 jobs - /// and 7 GB of max memory, each thread will use a maximum of 1 GB. - #[structopt(long, default_value = "7 GiB")] - pub max_memory: Byte, - - /// Size of the linked hash map cache when indexing. - /// The bigger it is, the faster the indexing is but the more memory it takes. - #[structopt(long, default_value = "500")] - pub linked_hash_map_size: usize, - - /// The name of the compression algorithm to use when compressing intermediate - /// chunks during indexing documents. - /// - /// Choosing a fast algorithm will make the indexing faster but may consume more memory. - #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] - pub chunk_compression_type: CompressionType, - - /// The level of compression of the chosen algorithm. - #[structopt(long, requires = "chunk-compression-type")] - pub chunk_compression_level: Option, - - /// The number of bytes to remove from the begining of the chunks while reading/sorting - /// or merging them. - /// - /// File fusing must only be enable on file systems that support the `FALLOC_FL_COLLAPSE_RANGE`, - /// (i.e. ext4 and XFS). File fusing will only work if the `enable-chunk-fusing` is set. - #[structopt(long, default_value = "4 GiB")] - pub chunk_fusing_shrink_size: Byte, - - /// Enable the chunk fusing or not, this reduces the amount of disk used by a factor of 2. - #[structopt(long)] - pub enable_chunk_fusing: bool, - - /// Number of parallel jobs for indexing, defaults to # of CPUs. - #[structopt(long)] - pub indexing_jobs: Option, -} - -struct UpdateHandler { - indexes: Arc, - max_nb_chunks: Option, - chunk_compression_level: Option, - thread_pool: ThreadPool, - log_frequency: usize, - max_memory: usize, - linked_hash_map_size: usize, - chunk_compression_type: CompressionType, - chunk_fusing_shrink_size: u64, -} - -impl UpdateHandler { - fn new( - opt: &IndexerOpts, - indexes: Arc, - ) -> Result { - let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(opt.indexing_jobs.unwrap_or(0)) - .build()?; - Ok(Self { - indexes, - max_nb_chunks: opt.max_nb_chunks, - chunk_compression_level: opt.chunk_compression_level, - thread_pool, - log_frequency: opt.log_every_n, - max_memory: opt.max_memory.get_bytes() as usize, - linked_hash_map_size: opt.linked_hash_map_size, - chunk_compression_type: opt.chunk_compression_type, - chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), - }) - } - - fn update_buidler(&self, update_id: u64) -> UpdateBuilder { - // We prepare the update by using the update builder. - let mut update_builder = UpdateBuilder::new(update_id); - if let Some(max_nb_chunks) = self.max_nb_chunks { - update_builder.max_nb_chunks(max_nb_chunks); - } - if let Some(chunk_compression_level) = self.chunk_compression_level { - update_builder.chunk_compression_level(chunk_compression_level); - } - update_builder.thread_pool(&self.thread_pool); - update_builder.log_every_n(self.log_frequency); - update_builder.max_memory(self.max_memory); - update_builder.linked_hash_map_size(self.linked_hash_map_size); - update_builder.chunk_compression_type(self.chunk_compression_type); - update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); - update_builder - } - - fn update_documents( - &self, - format: UpdateFormat, - method: IndexDocumentsMethod, - content: &[u8], - update_builder: UpdateBuilder, - ) -> Result { - // We must use the write transaction of the update here. - let mut wtxn = self.indexes.write_txn()?; - let mut builder = update_builder.index_documents(&mut wtxn, &self.indexes); - builder.update_format(format); - builder.index_documents_method(method); - - let gzipped = true; - let reader = if gzipped { - Box::new(GzDecoder::new(content)) - } else { - Box::new(content) as Box - }; - - let result = builder.execute(reader, |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } - - fn clear_documents(&self, update_builder: UpdateBuilder) -> Result { - // We must use the write transaction of the update here. - let mut wtxn = self.indexes.write_txn()?; - let builder = update_builder.clear_documents(&mut wtxn, &self.indexes); - - match builder.execute() { - Ok(_count) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } - - fn update_settings(&self, settings: &Settings, update_builder: UpdateBuilder) -> Result { - // We must use the write transaction of the update here. - let mut wtxn = self.indexes.write_txn()?; - let mut builder = update_builder.settings(&mut wtxn, &self.indexes); - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.searchable_attributes { - match names { - Some(names) => builder.set_searchable_fields(names.clone()), - None => builder.reset_searchable_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.displayed_attributes { - match names { - Some(names) => builder.set_displayed_fields(names.clone()), - None => builder.reset_displayed_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); - builder.set_faceted_fields(facet_types); - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { - match criteria { - Some(criteria) => builder.set_criteria(criteria.clone()), - None => builder.reset_criteria(), - } - } - - let result = builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } - - fn update_facets(&self, levels: &Facets, update_builder: UpdateBuilder) -> Result { - // We must use the write transaction of the update here. - let mut wtxn = self.indexes.write_txn()?; - let mut builder = update_builder.facets(&mut wtxn, &self.indexes); - if let Some(value) = levels.level_group_size { - builder.level_group_size(value); - } - if let Some(value) = levels.min_level_size { - builder.min_level_size(value); - } - match builder.execute() { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } -} - -impl Handler for UpdateHandler { - fn handle_update( - &mut self, - update_id: u64, - meta: Processing, - content: &[u8] - ) -> Result, Failed> { - use UpdateMeta::*; - - let update_builder = self.update_buidler(update_id); - - let result = match meta.meta() { - DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), - ClearDocuments => self.clear_documents(update_builder), - Settings(settings) => self.update_settings(settings, update_builder), - Facets(levels) => self.update_facets(levels, update_builder), - }; - - match result { - Ok(result) => Ok(meta.process(result)), - Err(e) => Err(meta.fail(e.to_string())), - } - } -} - -impl UpdateQueue { - pub fn new( - opt: &Opt, - indexes: Arc, - ) -> Result { - let handler = UpdateHandler::new(&opt.indexer_options, indexes)?; - let size = opt.max_udb_size.get_bytes() as usize; - let path = opt.db_path.join("updates.mdb"); - create_dir_all(&path)?; - let inner = UpdateStore::open( - Some(size), - path, - handler - )?; - Ok(Self { inner }) - } - - #[inline] - pub fn get_update_status(&self, update_id: u64) -> Result> { - Ok(self.inner.meta(update_id)?) - } -} diff --git a/src/updates/settings.rs b/src/updates/settings.rs deleted file mode 100644 index 91381edc5..000000000 --- a/src/updates/settings.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::num::NonZeroUsize; -use std::collections::HashMap; - -use serde::{Serialize, Deserialize, de::Deserializer}; - -// Any value that is present is considered Some value, including null. -fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> -where T: Deserialize<'de>, - D: Deserializer<'de> -{ - Deserialize::deserialize(deserializer).map(Some) -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct Settings { - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub displayed_attributes: Option>>, - - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub searchable_attributes: Option>>, - - #[serde(default)] - pub faceted_attributes: Option>>, - - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub criteria: Option>>, -} - -impl Settings { - pub fn cleared() -> Self { - Self { - displayed_attributes: Some(None), - searchable_attributes: Some(None), - faceted_attributes: Some(None), - criteria: Some(None), - } - } -} - - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct Facets { - pub level_group_size: Option, - pub min_level_size: Option, -} diff --git a/src/updates/update_store.rs b/src/updates/update_store.rs deleted file mode 100644 index f750fc38b..000000000 --- a/src/updates/update_store.rs +++ /dev/null @@ -1,581 +0,0 @@ -use std::path::Path; -use std::sync::{Arc, RwLock}; - -use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; -use heed::{EnvOpenOptions, Env, Database}; -use serde::{Serialize, Deserialize}; -use chrono::{DateTime, Utc}; - -type BEU64 = heed::zerocopy::U64; - -#[derive(Clone)] -pub struct UpdateStore { - env: Env, - pending_meta: Database, SerdeJson>>, - pending: Database, ByteSlice>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, - processing: Arc>>>, - notification_sender: Sender<()>, -} - -pub trait UpdateHandler { - fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed>; -} - -impl UpdateHandler for F -where F: FnMut(u64, Processing, &[u8]) -> Result, Failed> + Send + 'static { - fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed> { - self(update_id, meta, content) - } -} - -impl UpdateStore { - pub fn open( - size: Option, - path: P, - mut update_handler: U, - ) -> heed::Result>> - where - P: AsRef, - U: UpdateHandler + Send + 'static, - M: for<'a> Deserialize<'a> + Serialize + Send + Sync + Clone, - N: Serialize, - E: Serialize, - { - let mut options = EnvOpenOptions::new(); - if let Some(size) = size { - options.map_size(size); - } - options.max_dbs(5); - - let env = options.open(path)?; - let pending_meta = env.create_database(Some("pending-meta"))?; - let pending = env.create_database(Some("pending"))?; - let processed_meta = env.create_database(Some("processed-meta"))?; - let aborted_meta = env.create_database(Some("aborted-meta"))?; - let failed_meta = env.create_database(Some("failed-meta"))?; - let processing = Arc::new(RwLock::new(None)); - - let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); - // Send a first notification to trigger the process. - let _ = notification_sender.send(()); - - let update_store = Arc::new(UpdateStore { - env, - pending, - pending_meta, - processed_meta, - aborted_meta, - notification_sender, - failed_meta, - processing, - }); - - let update_store_cloned = update_store.clone(); - std::thread::spawn(move || { - // Block and wait for something to process. - for () in notification_receiver { - loop { - match update_store_cloned.process_pending_update(&mut update_handler) { - Ok(Some(_)) => (), - Ok(None) => break, - Err(e) => eprintln!("error while processing update: {}", e), - } - } - } - }); - - Ok(update_store) - } - - /// Returns the new biggest id to use to store the new update. - fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { - let last_pending = self.pending_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_processed = self.processed_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_aborted = self.aborted_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_update_id = [last_pending, last_processed, last_aborted] - .iter() - .copied() - .flatten() - .max(); - - match last_update_id { - Some(last_id) => Ok(last_id + 1), - None => Ok(0), - } - } - - /// Registers the update content in the pending store and the meta - /// into the pending-meta store. Returns the new unique update id. - pub fn register_update(&self, meta: M, content: &[u8]) -> heed::Result> - where M: Serialize, - { - let mut wtxn = self.env.write_txn()?; - - // We ask the update store to give us a new update id, this is safe, - // no other update can have the same id because we use a write txn before - // asking for the id and registering it so other update registering - // will be forced to wait for a new write txn. - let update_id = self.new_update_id(&wtxn)?; - let update_key = BEU64::new(update_id); - - let meta = Pending::new(meta, update_id); - self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, content)?; - - wtxn.commit()?; - - if let Err(e) = self.notification_sender.try_send(()) { - assert!(!e.is_disconnected(), "update notification channel is disconnected"); - } - Ok(meta) - } - /// Executes the user provided function on the next pending update (the one with the lowest id). - /// This is asynchronous as it let the user process the update with a read-only txn and - /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, handler: &mut U) -> heed::Result> - where - U: UpdateHandler, - M: for<'a> Deserialize<'a> + Serialize + Clone, - N: Serialize, - E: Serialize, - { - // Create a read transaction to be able to retrieve the pending update in order. - let rtxn = self.env.read_txn()?; - let first_meta = self.pending_meta.first(&rtxn)?; - - // If there is a pending update we process and only keep - // a reader while processing it, not a writer. - match first_meta { - Some((first_id, pending)) => { - let first_content = self.pending - .get(&rtxn, &first_id)? - .expect("associated update content"); - - // we cahnge the state of the update from pending to processing before we pass it - // to the update handler. Processing store is non persistent to be able recover - // from a failure - let processing = pending.processing(); - self.processing - .write() - .unwrap() - .replace(processing.clone()); - // Process the pending update using the provided user function. - let result = handler.handle_update(first_id.get(), processing, first_content); - drop(rtxn); - - // Once the pending update have been successfully processed - // we must remove the content from the pending and processing stores and - // write the *new* meta to the processed-meta store and commit. - let mut wtxn = self.env.write_txn()?; - self.processing - .write() - .unwrap() - .take(); - self.pending_meta.delete(&mut wtxn, &first_id)?; - self.pending.delete(&mut wtxn, &first_id)?; - match result { - Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, - Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, - } - wtxn.commit()?; - - Ok(Some(())) - }, - None => Ok(None) - } - } - - /// The id and metadata of the update that is currently being processed, - /// `None` if no update is being processed. - pub fn processing_update(&self) -> heed::Result)>> - where M: for<'a> Deserialize<'a>, - { - let rtxn = self.env.read_txn()?; - match self.pending_meta.first(&rtxn)? { - Some((key, meta)) => Ok(Some((key.get(), meta))), - None => Ok(None), - } - } - - /// Execute the user defined function with the meta-store iterators, the first - /// iterator is the *processed* meta one, the second the *aborted* meta one - /// and, the last is the *pending* meta one. - pub fn iter_metas(&self, mut f: F) -> heed::Result - where - M: for<'a> Deserialize<'a> + Clone, - N: for<'a> Deserialize<'a>, - F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - ) -> heed::Result, - { - let rtxn = self.env.read_txn()?; - - // We get the pending, processed and aborted meta iterators. - let processed_iter = self.processed_meta.iter(&rtxn)?; - let aborted_iter = self.aborted_meta.iter(&rtxn)?; - let pending_iter = self.pending_meta.iter(&rtxn)?; - let processing = self.processing.read().unwrap().clone(); - let failed_iter = self.failed_meta.iter(&rtxn)?; - - // We execute the user defined function with both iterators. - (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) - } - - /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta(&self, update_id: u64) -> heed::Result>> - where - M: for<'a> Deserialize<'a> + Clone, - N: for<'a> Deserialize<'a>, - E: for<'a> Deserialize<'a>, - { - let rtxn = self.env.read_txn()?; - let key = BEU64::new(update_id); - - if let Some(ref meta) = *self.processing.read().unwrap() { - if meta.id() == update_id { - return Ok(Some(UpdateStatus::Processing(meta.clone()))); - } - } - - println!("pending"); - if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Pending(meta))); - } - - println!("processed"); - if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Processed(meta))); - } - - if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Aborted(meta))); - } - - if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Failed(meta))); - } - - Ok(None) - } - - /// Aborts an update, an aborted update content is deleted and - /// the meta of it is moved into the aborted updates database. - /// - /// Trying to abort an update that is currently being processed, an update - /// that as already been processed or which doesn't actually exist, will - /// return `None`. - pub fn abort_update(&self, update_id: u64) -> heed::Result>> - where M: Serialize + for<'a> Deserialize<'a>, - { - let mut wtxn = self.env.write_txn()?; - let key = BEU64::new(update_id); - - // We cannot abort an update that is currently being processed. - if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { - return Ok(None); - } - - let pending = match self.pending_meta.get(&wtxn, &key)? { - Some(meta) => meta, - None => return Ok(None), - }; - - let aborted = pending.abort(); - - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - - wtxn.commit()?; - - Ok(Some(aborted)) - } - - /// Aborts all the pending updates, and not the one being currently processed. - /// Returns the update metas and ids that were successfully aborted. - pub fn abort_pendings(&self) -> heed::Result)>> - where M: Serialize + for<'a> Deserialize<'a>, - { - let mut wtxn = self.env.write_txn()?; - let mut aborted_updates = Vec::new(); - - // We skip the first pending update as it is currently being processed. - for result in self.pending_meta.iter(&wtxn)?.skip(1) { - let (key, pending) = result?; - let id = key.get(); - aborted_updates.push((id, pending.abort())); - } - - for (id, aborted) in &aborted_updates { - let key = BEU64::new(*id); - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - } - - wtxn.commit()?; - - Ok(aborted_updates) - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Pending { - update_id: u64, - meta: M, - enqueued_at: DateTime, -} - -impl Pending { - fn new(meta: M, update_id: u64) -> Self { - Self { - enqueued_at: Utc::now(), - meta, - update_id, - } - } - - pub fn processing(self) -> Processing { - Processing { - from: self, - started_processing_at: Utc::now(), - } - } - - pub fn abort(self) -> Aborted { - Aborted { - from: self, - aborted_at: Utc::now(), - } - } - - pub fn meta(&self) -> &M { - &self.meta - } - - pub fn id(&self) -> u64 { - self.update_id - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Processed { - success: N, - processed_at: DateTime, - #[serde(flatten)] - from: Processing, -} - -impl Processed { - fn id(&self) -> u64 { - self.from.id() - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Processing { - #[serde(flatten)] - from: Pending, - started_processing_at: DateTime, -} - -impl Processing { - pub fn id(&self) -> u64 { - self.from.id() - } - - pub fn meta(&self) -> &M { - self.from.meta() - } - - pub fn process(self, meta: N) -> Processed { - Processed { - success: meta, - from: self, - processed_at: Utc::now(), - } - } - - pub fn fail(self, error: E) -> Failed { - Failed { - from: self, - error, - failed_at: Utc::now(), - } - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Aborted { - #[serde(flatten)] - from: Pending, - aborted_at: DateTime, -} - -impl Aborted { - fn id(&self) -> u64 { - self.from.id() - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Failed { - #[serde(flatten)] - from: Processing, - error: E, - failed_at: DateTime, -} - -impl Failed { - fn id(&self) -> u64 { - self.from.id() - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize)] -#[serde(tag = "status")] -pub enum UpdateStatus { - Processing(Processing), - Pending(Pending), - Processed(Processed), - Aborted(Aborted), - Failed(Failed), -} - -impl UpdateStatus { - pub fn id(&self) -> u64 { - match self { - UpdateStatus::Processing(u) => u.id(), - UpdateStatus::Pending(u) => u.id(), - UpdateStatus::Processed(u) => u.id(), - UpdateStatus::Aborted(u) => u.id(), - UpdateStatus::Failed(u) => u.id(), - } - } -} - -impl From> for UpdateStatus { - fn from(other: Pending) -> Self { - Self::Pending(other) - } -} - -impl From> for UpdateStatus { - fn from(other: Aborted) -> Self { - Self::Aborted(other) - } -} - -impl From> for UpdateStatus { - fn from(other: Processed) -> Self { - Self::Processed(other) - } -} - -impl From> for UpdateStatus { - fn from(other: Processing) -> Self { - Self::Processing(other) - } -} - -impl From> for UpdateStatus { - fn from(other: Failed) -> Self { - Self::Failed(other) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::thread; - use std::time::{Duration, Instant}; - - #[test] - fn simple() { - let dir = tempfile::tempdir().unwrap(); - let update_store = UpdateStore::open(None, dir, |_id, meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let meta = String::from("kiki"); - let update = update_store.register_update(meta, &[]).unwrap(); - thread::sleep(Duration::from_millis(100)); - let meta = update_store.meta(update.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - } - - #[test] - #[ignore] - fn long_running_update() { - let dir = tempfile::tempdir().unwrap(); - let update_store = UpdateStore::open(None, dir, |_id, meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let before_register = Instant::now(); - - let meta = String::from("kiki"); - let update_kiki = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("coco"); - let update_coco = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("cucu"); - let update_cucu = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - thread::sleep(Duration::from_millis(400 * 3 + 100)); - - let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "coco processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "cucu processed"); - } else { - panic!() - } - } -} From da056a687794caf5f9a3725c14fe14b45184e9b4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 20:55:29 +0100 Subject: [PATCH 026/522] comment tests out --- .../local_index_controller/index_store.rs | 6 +- .../local_index_controller/update_handler.rs | 4 +- .../local_index_controller/update_store.rs | 144 +- src/index_controller/updates.rs | 16 +- tests/assets/dumps/v1/metadata.json | 12 - tests/assets/dumps/v1/test/documents.jsonl | 77 - tests/assets/dumps/v1/test/settings.json | 59 - tests/assets/dumps/v1/test/updates.jsonl | 2 - tests/assets/test_set.json | 1613 -------------- tests/common.rs | 569 ----- tests/dashboard.rs | 12 - tests/documents_add.rs | 222 -- tests/documents_delete.rs | 67 - tests/documents_get.rs | 23 - tests/dump.rs | 395 ---- tests/errors.rs | 200 -- tests/health.rs | 11 - tests/index.rs | 809 ------- tests/index_update.rs | 200 -- tests/lazy_index_creation.rs | 446 ---- tests/placeholder_search.rs | 629 ------ tests/search.rs | 1879 ----------------- tests/search_settings.rs | 538 ----- tests/settings.rs | 523 ----- tests/settings_ranking_rules.rs | 182 -- tests/settings_stop_words.rs | 61 - tests/url_normalizer.rs | 18 - 27 files changed, 132 insertions(+), 8585 deletions(-) delete mode 100644 tests/assets/dumps/v1/metadata.json delete mode 100644 tests/assets/dumps/v1/test/documents.jsonl delete mode 100644 tests/assets/dumps/v1/test/settings.json delete mode 100644 tests/assets/dumps/v1/test/updates.jsonl delete mode 100644 tests/assets/test_set.json delete mode 100644 tests/common.rs delete mode 100644 tests/dashboard.rs delete mode 100644 tests/documents_add.rs delete mode 100644 tests/documents_delete.rs delete mode 100644 tests/documents_get.rs delete mode 100644 tests/dump.rs delete mode 100644 tests/errors.rs delete mode 100644 tests/health.rs delete mode 100644 tests/index.rs delete mode 100644 tests/index_update.rs delete mode 100644 tests/lazy_index_creation.rs delete mode 100644 tests/placeholder_search.rs delete mode 100644 tests/search.rs delete mode 100644 tests/search_settings.rs delete mode 100644 tests/settings.rs delete mode 100644 tests/settings_ranking_rules.rs delete mode 100644 tests/settings_stop_words.rs delete mode 100644 tests/url_normalizer.rs diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 4c9a98472..b94d9e492 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -11,9 +11,11 @@ use uuid::Uuid; use serde::{Serialize, Deserialize}; use log::warn; -use super::update_store::UpdateStore; -use super::update_handler::UpdateHandler; use crate::option::IndexerOpts; +use super::update_handler::UpdateHandler; +use super::{UpdateMeta, UpdateResult}; + +type UpdateStore = super::update_store::UpdateStore; #[derive(Serialize, Deserialize, Debug)] struct IndexMeta { diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index fae3ad0ae..c89b9f686 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -182,12 +182,13 @@ impl UpdateHandler { impl HandleUpdate for UpdateHandler { fn handle_update( &mut self, - update_id: u64, meta: Processing, content: &[u8] ) -> Result, Failed> { use UpdateMeta::*; + let update_id = meta.id(); + let update_builder = self.update_buidler(update_id); let result = match meta.meta() { @@ -203,4 +204,3 @@ impl HandleUpdate for UpdateHandler { } } } - diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs index 94543b0a1..9a17ec00f 100644 --- a/src/index_controller/local_index_controller/update_store.rs +++ b/src/index_controller/local_index_controller/update_store.rs @@ -4,37 +4,42 @@ use std::sync::{Arc, RwLock}; use crossbeam_channel::Sender; use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; use heed::{EnvOpenOptions, Env, Database}; +use serde::{Serialize, Deserialize}; use crate::index_controller::updates::*; -use crate::index_controller::{UpdateMeta, UpdateResult}; type BEU64 = heed::zerocopy::U64; #[derive(Clone)] -pub struct UpdateStore { +pub struct UpdateStore { env: Env, - pending_meta: Database, SerdeJson>>, + pending_meta: Database, SerdeJson>>, pending: Database, ByteSlice>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, - processing: Arc>>>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, notification_sender: Sender<()>, } pub trait HandleUpdate { - fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed>; + fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed>; } -impl UpdateStore { +impl UpdateStore +where + M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, + N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, + E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, +{ pub fn open( mut options: EnvOpenOptions, path: P, mut update_handler: U, - ) -> heed::Result> + ) -> heed::Result> where P: AsRef, - U: HandleUpdate + Send + 'static, + U: HandleUpdate + Send + 'static, { options.max_dbs(5); @@ -111,9 +116,9 @@ impl UpdateStore { /// into the pending-meta store. Returns the new unique update id. pub fn register_update( &self, - meta: UpdateMeta, + meta: M, content: &[u8] - ) -> heed::Result> { + ) -> heed::Result> { let mut wtxn = self.env.write_txn()?; // We ask the update store to give us a new update id, this is safe, @@ -139,7 +144,7 @@ impl UpdateStore { /// only writing the result meta to the processed-meta store *after* it has been processed. fn process_pending_update(&self, handler: &mut U) -> heed::Result> where - U: HandleUpdate + Send + 'static, + U: HandleUpdate + Send + 'static, { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; @@ -153,7 +158,7 @@ impl UpdateStore { .get(&rtxn, &first_id)? .expect("associated update content"); - // we cahnge the state of the update from pending to processing before we pass it + // we change the state of the update from pending to processing before we pass it // to the update handler. Processing store is non persistent to be able recover // from a failure let processing = pending.processing(); @@ -162,7 +167,7 @@ impl UpdateStore { .unwrap() .replace(processing.clone()); // Process the pending update using the provided user function. - let result = handler.handle_update(first_id.get(), processing, first_content); + let result = handler.handle_update(processing, first_content); drop(rtxn); // Once the pending update have been successfully processed @@ -189,10 +194,10 @@ impl UpdateStore { /// The id and metadata of the update that is currently being processed, /// `None` if no update is being processed. - pub fn processing_update(&self) -> heed::Result)>> { + pub fn processing_update(&self) -> heed::Result>> { let rtxn = self.env.read_txn()?; match self.pending_meta.first(&rtxn)? { - Some((key, meta)) => Ok(Some((key.get(), meta))), + Some((_, meta)) => Ok(Some(meta)), None => Ok(None), } } @@ -203,11 +208,11 @@ impl UpdateStore { pub fn iter_metas(&self, mut f: F) -> heed::Result where F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, ) -> heed::Result, { let rtxn = self.env.read_txn()?; @@ -224,7 +229,7 @@ impl UpdateStore { } /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta(&self, update_id: u64) -> heed::Result>> { + pub fn meta(&self, update_id: u64) -> heed::Result>> { let rtxn = self.env.read_txn()?; let key = BEU64::new(update_id); @@ -259,7 +264,7 @@ impl UpdateStore { /// Trying to abort an update that is currently being processed, an update /// that as already been processed or which doesn't actually exist, will /// return `None`. - pub fn abort_update(&self, update_id: u64) -> heed::Result>> { + pub fn abort_update(&self, update_id: u64) -> heed::Result>> { let mut wtxn = self.env.write_txn()?; let key = BEU64::new(update_id); @@ -286,7 +291,7 @@ impl UpdateStore { /// Aborts all the pending updates, and not the one being currently processed. /// Returns the update metas and ids that were successfully aborted. - pub fn abort_pendings(&self) -> heed::Result)>> { + pub fn abort_pendings(&self) -> heed::Result)>> { let mut wtxn = self.env.write_txn()?; let mut aborted_updates = Vec::new(); @@ -309,3 +314,90 @@ impl UpdateStore { Ok(aborted_updates) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::{Duration, Instant}; + + impl HandleUpdate for F + where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { + fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { + self(meta, content) + } + } + + #[test] + fn simple() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let meta = String::from("kiki"); + let update = update_store.register_update(meta, &[]).unwrap(); + thread::sleep(Duration::from_millis(100)); + let meta = update_store.meta(update.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + } + + #[test] + #[ignore] + fn long_running_update() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let before_register = Instant::now(); + + let meta = String::from("kiki"); + let update_kiki = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("coco"); + let update_coco = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("cucu"); + let update_cucu = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + thread::sleep(Duration::from_millis(400 * 3 + 100)); + + let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "coco processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "cucu processed"); + } else { + panic!() + } + } +} diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs index 7c67ea8c2..4eb94dc4a 100644 --- a/src/index_controller/updates.rs +++ b/src/index_controller/updates.rs @@ -3,9 +3,9 @@ use serde::{Serialize, Deserialize}; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] pub struct Pending { - update_id: u64, - meta: M, - enqueued_at: DateTime, + pub update_id: u64, + pub meta: M, + pub enqueued_at: DateTime, } impl Pending { @@ -42,10 +42,10 @@ impl Pending { #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] pub struct Processed { - success: N, - processed_at: DateTime, + pub success: N, + pub processed_at: DateTime, #[serde(flatten)] - from: Processing, + pub from: Processing, } impl Processed { @@ -57,8 +57,8 @@ impl Processed { #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] pub struct Processing { #[serde(flatten)] - from: Pending, - started_processing_at: DateTime, + pub from: Pending, + pub started_processing_at: DateTime, } impl Processing { diff --git a/tests/assets/dumps/v1/metadata.json b/tests/assets/dumps/v1/metadata.json deleted file mode 100644 index 6fe302324..000000000 --- a/tests/assets/dumps/v1/metadata.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "indices": [{ - "uid": "test", - "primaryKey": "id" - }, { - "uid": "test2", - "primaryKey": "test2_id" - } - ], - "dbVersion": "0.13.0", - "dumpVersion": "1" -} diff --git a/tests/assets/dumps/v1/test/documents.jsonl b/tests/assets/dumps/v1/test/documents.jsonl deleted file mode 100644 index 7af80f342..000000000 --- a/tests/assets/dumps/v1/test/documents.jsonl +++ /dev/null @@ -1,77 +0,0 @@ -{"id":0,"isActive":false,"balance":"$2,668.55","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Lucas Hess","gender":"male","email":"lucashess@chorizon.com","phone":"+1 (998) 478-2597","address":"412 Losee Terrace, Blairstown, Georgia, 2825","about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n","registered":"2016-06-21T09:30:25 -02:00","latitude":-44.174957,"longitude":-145.725388,"tags":["bug","bug"]} -{"id":1,"isActive":true,"balance":"$1,706.13","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Cherry Orr","gender":"female","email":"cherryorr@chorizon.com","phone":"+1 (995) 479-3174","address":"442 Beverly Road, Ventress, New Mexico, 3361","about":"Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n","registered":"2020-03-18T11:12:21 -01:00","latitude":-24.356932,"longitude":27.184808,"tags":["new issue","bug"]} -{"id":2,"isActive":true,"balance":"$2,467.47","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Patricia Goff","gender":"female","email":"patriciagoff@chorizon.com","phone":"+1 (864) 463-2277","address":"866 Hornell Loop, Cresaptown, Ohio, 1700","about":"Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n","registered":"2014-10-28T12:59:30 -01:00","latitude":-64.008555,"longitude":11.867098,"tags":["good first issue"]} -{"id":3,"isActive":true,"balance":"$3,344.40","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Adeline Flynn","gender":"female","email":"adelineflynn@chorizon.com","phone":"+1 (994) 600-2840","address":"428 Paerdegat Avenue, Hollymead, Pennsylvania, 948","about":"Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n","registered":"2014-03-27T06:24:45 -01:00","latitude":-74.485173,"longitude":-11.059859,"tags":["bug","good first issue","wontfix","new issue"]} -{"id":4,"isActive":false,"balance":"$2,575.78","picture":"http://placehold.it/32x32","age":39,"color":"Green","name":"Mariana Pacheco","gender":"female","email":"marianapacheco@chorizon.com","phone":"+1 (820) 414-2223","address":"664 Rapelye Street, Faywood, California, 7320","about":"Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n","registered":"2015-09-02T03:23:35 -02:00","latitude":75.763501,"longitude":-78.777124,"tags":["new issue"]} -{"id":5,"isActive":true,"balance":"$3,793.09","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Warren Watson","gender":"male","email":"warrenwatson@chorizon.com","phone":"+1 (807) 583-2427","address":"671 Prince Street, Faxon, Connecticut, 4275","about":"Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n","registered":"2017-06-04T06:02:17 -02:00","latitude":29.979223,"longitude":25.358943,"tags":["wontfix","wontfix","wontfix"]} -{"id":6,"isActive":true,"balance":"$2,919.70","picture":"http://placehold.it/32x32","age":20,"color":"blue","name":"Shelia Berry","gender":"female","email":"sheliaberry@chorizon.com","phone":"+1 (853) 511-2651","address":"437 Forrest Street, Coventry, Illinois, 2056","about":"Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n","registered":"2018-07-11T02:45:01 -02:00","latitude":54.815991,"longitude":-118.690609,"tags":["good first issue","bug","wontfix","new issue"]} -{"id":7,"isActive":true,"balance":"$1,349.50","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Chrystal Boyd","gender":"female","email":"chrystalboyd@chorizon.com","phone":"+1 (936) 563-2802","address":"670 Croton Loop, Sussex, Florida, 4692","about":"Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n","registered":"2016-11-01T07:36:04 -01:00","latitude":-24.711933,"longitude":147.246705,"tags":[]} -{"id":8,"isActive":false,"balance":"$3,999.56","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Martin Porter","gender":"male","email":"martinporter@chorizon.com","phone":"+1 (895) 580-2304","address":"577 Regent Place, Aguila, Guam, 6554","about":"Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n","registered":"2014-09-20T02:08:30 -02:00","latitude":-88.344273,"longitude":37.964466,"tags":[]} -{"id":9,"isActive":true,"balance":"$3,729.71","picture":"http://placehold.it/32x32","age":26,"color":"blue","name":"Kelli Mendez","gender":"female","email":"kellimendez@chorizon.com","phone":"+1 (936) 401-2236","address":"242 Caton Place, Grazierville, Alabama, 3968","about":"Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n","registered":"2018-05-04T10:35:30 -02:00","latitude":49.37551,"longitude":41.872323,"tags":["new issue","new issue"]} -{"id":10,"isActive":false,"balance":"$1,127.47","picture":"http://placehold.it/32x32","age":27,"color":"blue","name":"Maddox Johns","gender":"male","email":"maddoxjohns@chorizon.com","phone":"+1 (892) 470-2357","address":"756 Beard Street, Avalon, Louisiana, 114","about":"Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n","registered":"2016-04-22T06:41:25 -02:00","latitude":66.640229,"longitude":-17.222666,"tags":["new issue","good first issue","good first issue","new issue"]} -{"id":11,"isActive":true,"balance":"$1,351.43","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Evans Wagner","gender":"male","email":"evanswagner@chorizon.com","phone":"+1 (889) 496-2332","address":"118 Monaco Place, Lutsen, Delaware, 6209","about":"Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n","registered":"2016-10-27T01:26:31 -02:00","latitude":-77.673222,"longitude":-142.657214,"tags":["good first issue","good first issue"]} -{"id":12,"isActive":false,"balance":"$3,394.96","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Aida Kirby","gender":"female","email":"aidakirby@chorizon.com","phone":"+1 (942) 532-2325","address":"797 Engert Avenue, Wilsonia, Idaho, 6532","about":"Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n","registered":"2018-06-18T04:39:57 -02:00","latitude":-58.062041,"longitude":34.999254,"tags":["new issue","wontfix","bug","new issue"]} -{"id":13,"isActive":true,"balance":"$2,812.62","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Nelda Burris","gender":"female","email":"neldaburris@chorizon.com","phone":"+1 (813) 600-2576","address":"160 Opal Court, Fowlerville, Tennessee, 2170","about":"Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n","registered":"2015-08-15T12:39:53 -02:00","latitude":66.6871,"longitude":179.549488,"tags":["wontfix"]} -{"id":14,"isActive":true,"balance":"$1,718.33","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Jennifer Hart","gender":"female","email":"jenniferhart@chorizon.com","phone":"+1 (850) 537-2513","address":"124 Veranda Place, Nash, Utah, 985","about":"Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n","registered":"2016-09-04T11:46:59 -02:00","latitude":-66.827751,"longitude":99.220079,"tags":["wontfix","bug","new issue","new issue"]} -{"id":15,"isActive":false,"balance":"$2,698.16","picture":"http://placehold.it/32x32","age":28,"color":"blue","name":"Aurelia Contreras","gender":"female","email":"aureliacontreras@chorizon.com","phone":"+1 (932) 442-3103","address":"655 Dwight Street, Grapeview, Palau, 8356","about":"Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n","registered":"2014-09-11T10:43:15 -02:00","latitude":-71.328973,"longitude":133.404895,"tags":["wontfix","bug","good first issue"]} -{"id":16,"isActive":true,"balance":"$3,303.25","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Estella Bass","gender":"female","email":"estellabass@chorizon.com","phone":"+1 (825) 436-2909","address":"435 Rockwell Place, Garberville, Wisconsin, 2230","about":"Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n","registered":"2017-11-23T09:32:09 -01:00","latitude":81.17014,"longitude":-145.262693,"tags":["new issue"]} -{"id":17,"isActive":false,"balance":"$3,579.20","picture":"http://placehold.it/32x32","age":25,"color":"brown","name":"Ortega Brennan","gender":"male","email":"ortegabrennan@chorizon.com","phone":"+1 (906) 526-2287","address":"440 Berry Street, Rivera, Maine, 1849","about":"Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n","registered":"2016-03-31T02:17:13 -02:00","latitude":-68.407524,"longitude":-113.642067,"tags":["new issue","wontfix"]} -{"id":18,"isActive":false,"balance":"$1,484.92","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Leonard Tillman","gender":"male","email":"leonardtillman@chorizon.com","phone":"+1 (864) 541-3456","address":"985 Provost Street, Charco, New Hampshire, 8632","about":"Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n","registered":"2018-05-06T08:21:27 -02:00","latitude":-8.581801,"longitude":-61.910062,"tags":["wontfix","new issue","bug","bug"]} -{"id":19,"isActive":true,"balance":"$3,572.55","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Dale Payne","gender":"male","email":"dalepayne@chorizon.com","phone":"+1 (814) 469-3499","address":"536 Dare Court, Ironton, Arkansas, 8605","about":"Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n","registered":"2019-10-11T01:01:33 -02:00","latitude":-18.280968,"longitude":-126.091797,"tags":["bug","wontfix","wontfix","wontfix"]} -{"id":20,"isActive":true,"balance":"$1,986.48","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Florence Long","gender":"female","email":"florencelong@chorizon.com","phone":"+1 (972) 557-3858","address":"519 Hendrickson Street, Templeton, Hawaii, 2389","about":"Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n","registered":"2016-05-02T09:18:59 -02:00","latitude":-27.110866,"longitude":-45.09445,"tags":[]} -{"id":21,"isActive":true,"balance":"$1,440.09","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Levy Whitley","gender":"male","email":"levywhitley@chorizon.com","phone":"+1 (911) 458-2411","address":"187 Thomas Street, Hachita, North Carolina, 2989","about":"Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n","registered":"2014-04-30T07:31:38 -02:00","latitude":-6.537315,"longitude":171.813536,"tags":["bug"]} -{"id":22,"isActive":false,"balance":"$2,938.57","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Bernard Mcfarland","gender":"male","email":"bernardmcfarland@chorizon.com","phone":"+1 (979) 442-3386","address":"409 Hall Street, Keyport, Federated States Of Micronesia, 7011","about":"Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n","registered":"2017-08-10T10:07:59 -02:00","latitude":63.766795,"longitude":68.177069,"tags":[]} -{"id":23,"isActive":true,"balance":"$1,678.49","picture":"http://placehold.it/32x32","age":31,"color":"brown","name":"Blanca Mcclain","gender":"female","email":"blancamcclain@chorizon.com","phone":"+1 (976) 439-2772","address":"176 Crooke Avenue, Valle, Virginia, 5373","about":"Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n","registered":"2015-10-12T11:57:28 -02:00","latitude":-8.944564,"longitude":-150.711709,"tags":["bug","wontfix","good first issue"]} -{"id":24,"isActive":true,"balance":"$2,276.87","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Espinoza Ford","gender":"male","email":"espinozaford@chorizon.com","phone":"+1 (945) 429-3975","address":"137 Bowery Street, Itmann, District Of Columbia, 1864","about":"Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n","registered":"2014-03-26T02:16:08 -01:00","latitude":-37.137666,"longitude":-51.811757,"tags":["wontfix","bug"]} -{"id":25,"isActive":true,"balance":"$3,973.43","picture":"http://placehold.it/32x32","age":29,"color":"Green","name":"Sykes Conley","gender":"male","email":"sykesconley@chorizon.com","phone":"+1 (851) 401-3916","address":"345 Grand Street, Woodlands, Missouri, 4461","about":"Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n","registered":"2015-09-12T06:03:56 -02:00","latitude":67.282955,"longitude":-64.341323,"tags":["wontfix"]} -{"id":26,"isActive":false,"balance":"$1,431.50","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Barlow Duran","gender":"male","email":"barlowduran@chorizon.com","phone":"+1 (995) 436-2562","address":"481 Everett Avenue, Allison, Nebraska, 3065","about":"Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n","registered":"2017-06-29T04:28:43 -02:00","latitude":-38.70606,"longitude":55.02816,"tags":["new issue"]} -{"id":27,"isActive":true,"balance":"$3,478.27","picture":"http://placehold.it/32x32","age":31,"color":"blue","name":"Schwartz Morgan","gender":"male","email":"schwartzmorgan@chorizon.com","phone":"+1 (861) 507-2067","address":"451 Lincoln Road, Fairlee, Washington, 2717","about":"Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n","registered":"2016-05-10T08:34:54 -02:00","latitude":-75.886403,"longitude":93.044471,"tags":["bug","bug","wontfix","wontfix"]} -{"id":28,"isActive":true,"balance":"$2,825.59","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Kristy Leon","gender":"female","email":"kristyleon@chorizon.com","phone":"+1 (948) 465-2563","address":"594 Macon Street, Floris, South Dakota, 3565","about":"Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n","registered":"2014-12-14T04:10:29 -01:00","latitude":-50.01615,"longitude":-68.908804,"tags":["wontfix","good first issue"]} -{"id":29,"isActive":false,"balance":"$3,028.03","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Ashley Pittman","gender":"male","email":"ashleypittman@chorizon.com","phone":"+1 (928) 507-3523","address":"646 Adelphi Street, Clara, Colorado, 6056","about":"Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n","registered":"2016-01-07T10:40:48 -01:00","latitude":-58.766037,"longitude":-124.828485,"tags":["wontfix"]} -{"id":30,"isActive":true,"balance":"$2,021.11","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Stacy Espinoza","gender":"female","email":"stacyespinoza@chorizon.com","phone":"+1 (999) 487-3253","address":"931 Alabama Avenue, Bangor, Alaska, 8215","about":"Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n","registered":"2014-07-16T06:15:53 -02:00","latitude":41.560197,"longitude":177.697,"tags":["new issue","new issue","bug"]} -{"id":31,"isActive":false,"balance":"$3,609.82","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Vilma Garza","gender":"female","email":"vilmagarza@chorizon.com","phone":"+1 (944) 585-2021","address":"565 Tech Place, Sedley, Puerto Rico, 858","about":"Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n","registered":"2017-06-30T07:43:52 -02:00","latitude":-12.574889,"longitude":-54.771186,"tags":["new issue","wontfix","wontfix"]} -{"id":32,"isActive":false,"balance":"$2,882.34","picture":"http://placehold.it/32x32","age":38,"color":"brown","name":"June Dunlap","gender":"female","email":"junedunlap@chorizon.com","phone":"+1 (997) 504-2937","address":"353 Cozine Avenue, Goodville, Indiana, 1438","about":"Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n","registered":"2016-08-23T08:54:11 -02:00","latitude":-27.883363,"longitude":-163.919683,"tags":["new issue","new issue","bug","wontfix"]} -{"id":33,"isActive":true,"balance":"$3,556.54","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Cecilia Greer","gender":"female","email":"ceciliagreer@chorizon.com","phone":"+1 (977) 573-3498","address":"696 Withers Street, Lydia, Oklahoma, 3220","about":"Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n","registered":"2017-01-13T11:30:12 -01:00","latitude":60.467215,"longitude":84.684575,"tags":["wontfix","good first issue","good first issue","wontfix"]} -{"id":34,"isActive":true,"balance":"$1,413.35","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Mckay Schroeder","gender":"male","email":"mckayschroeder@chorizon.com","phone":"+1 (816) 480-3657","address":"958 Miami Court, Rehrersburg, Northern Mariana Islands, 567","about":"Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n","registered":"2016-02-08T04:50:15 -01:00","latitude":-72.413287,"longitude":-159.254371,"tags":["good first issue"]} -{"id":35,"isActive":true,"balance":"$2,306.53","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Sawyer Mccormick","gender":"male","email":"sawyermccormick@chorizon.com","phone":"+1 (829) 569-3012","address":"749 Apollo Street, Eastvale, Texas, 7373","about":"Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n","registered":"2019-11-30T11:53:23 -01:00","latitude":-48.978194,"longitude":110.950191,"tags":["good first issue","new issue","new issue","bug"]} -{"id":36,"isActive":false,"balance":"$1,844.54","picture":"http://placehold.it/32x32","age":37,"color":"brown","name":"Barbra Valenzuela","gender":"female","email":"barbravalenzuela@chorizon.com","phone":"+1 (992) 512-2649","address":"617 Schenck Court, Reinerton, Michigan, 2908","about":"Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n","registered":"2019-03-29T01:59:31 -01:00","latitude":45.193723,"longitude":-12.486778,"tags":["new issue","new issue","wontfix","wontfix"]} -{"id":37,"isActive":false,"balance":"$3,469.82","picture":"http://placehold.it/32x32","age":39,"color":"brown","name":"Opal Weiss","gender":"female","email":"opalweiss@chorizon.com","phone":"+1 (809) 400-3079","address":"535 Bogart Street, Frizzleburg, Arizona, 5222","about":"Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n","registered":"2019-09-04T07:22:28 -02:00","latitude":72.50376,"longitude":61.656435,"tags":["bug","bug","good first issue","good first issue"]} -{"id":38,"isActive":true,"balance":"$1,992.38","picture":"http://placehold.it/32x32","age":40,"color":"Green","name":"Christina Short","gender":"female","email":"christinashort@chorizon.com","phone":"+1 (884) 589-2705","address":"594 Willmohr Street, Dexter, Montana, 660","about":"Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n","registered":"2014-01-21T09:31:56 -01:00","latitude":-42.762739,"longitude":77.052349,"tags":["bug","new issue"]} -{"id":39,"isActive":false,"balance":"$1,722.85","picture":"http://placehold.it/32x32","age":29,"color":"brown","name":"Golden Horton","gender":"male","email":"goldenhorton@chorizon.com","phone":"+1 (903) 426-2489","address":"191 Schenck Avenue, Mayfair, North Dakota, 5000","about":"Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n","registered":"2015-08-19T02:56:41 -02:00","latitude":69.922534,"longitude":9.881433,"tags":["bug"]} -{"id":40,"isActive":false,"balance":"$1,656.54","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Stafford Emerson","gender":"male","email":"staffordemerson@chorizon.com","phone":"+1 (992) 455-2573","address":"523 Thornton Street, Conway, Vermont, 6331","about":"Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n","registered":"2019-02-16T04:07:08 -01:00","latitude":-29.143111,"longitude":-57.207703,"tags":["wontfix","good first issue","good first issue"]} -{"id":41,"isActive":false,"balance":"$1,861.56","picture":"http://placehold.it/32x32","age":21,"color":"brown","name":"Salinas Gamble","gender":"male","email":"salinasgamble@chorizon.com","phone":"+1 (901) 525-2373","address":"991 Nostrand Avenue, Kansas, Mississippi, 6756","about":"Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n","registered":"2017-08-21T05:47:53 -02:00","latitude":-22.593819,"longitude":-63.613004,"tags":["good first issue","bug","bug","wontfix"]} -{"id":42,"isActive":true,"balance":"$3,179.74","picture":"http://placehold.it/32x32","age":34,"color":"brown","name":"Graciela Russell","gender":"female","email":"gracielarussell@chorizon.com","phone":"+1 (893) 464-3951","address":"361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713","about":"Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n","registered":"2015-05-18T09:52:56 -02:00","latitude":-14.634444,"longitude":12.931783,"tags":["wontfix","bug","wontfix"]} -{"id":43,"isActive":true,"balance":"$1,777.38","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Arnold Bender","gender":"male","email":"arnoldbender@chorizon.com","phone":"+1 (945) 581-3808","address":"781 Lorraine Street, Gallina, American Samoa, 1832","about":"Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n","registered":"2018-12-23T02:26:30 -01:00","latitude":41.208579,"longitude":51.948925,"tags":["bug","good first issue","good first issue","wontfix"]} -{"id":44,"isActive":true,"balance":"$2,893.45","picture":"http://placehold.it/32x32","age":22,"color":"Green","name":"Joni Spears","gender":"female","email":"jonispears@chorizon.com","phone":"+1 (916) 565-2124","address":"307 Harwood Place, Canterwood, Maryland, 2047","about":"Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n","registered":"2015-03-01T12:38:28 -01:00","latitude":8.19071,"longitude":146.323808,"tags":["wontfix","new issue","good first issue","good first issue"]} -{"id":45,"isActive":true,"balance":"$2,830.36","picture":"http://placehold.it/32x32","age":20,"color":"brown","name":"Irene Bennett","gender":"female","email":"irenebennett@chorizon.com","phone":"+1 (904) 431-2211","address":"353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686","about":"Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n","registered":"2018-04-17T05:18:51 -02:00","latitude":-36.435177,"longitude":-127.552573,"tags":["bug","wontfix"]} -{"id":46,"isActive":true,"balance":"$1,348.04","picture":"http://placehold.it/32x32","age":34,"color":"Green","name":"Lawson Curtis","gender":"male","email":"lawsoncurtis@chorizon.com","phone":"+1 (896) 532-2172","address":"942 Gerritsen Avenue, Southmont, Kansas, 8915","about":"Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n","registered":"2016-08-23T01:41:09 -02:00","latitude":-48.783539,"longitude":20.492944,"tags":[]} -{"id":47,"isActive":true,"balance":"$1,132.41","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goff May","gender":"male","email":"goffmay@chorizon.com","phone":"+1 (859) 453-3415","address":"225 Rutledge Street, Boonville, Massachusetts, 4081","about":"Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n","registered":"2014-10-25T07:32:30 -02:00","latitude":13.079225,"longitude":76.215086,"tags":["bug"]} -{"id":48,"isActive":true,"balance":"$1,201.87","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goodman Becker","gender":"male","email":"goodmanbecker@chorizon.com","phone":"+1 (825) 470-3437","address":"388 Seigel Street, Sisquoc, Kentucky, 8231","about":"Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n","registered":"2019-09-05T04:49:03 -02:00","latitude":-23.792094,"longitude":-13.621221,"tags":["bug","bug","wontfix","new issue"]} -{"id":49,"isActive":true,"balance":"$1,476.39","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Maureen Dale","gender":"female","email":"maureendale@chorizon.com","phone":"+1 (984) 538-3684","address":"817 Newton Street, Bannock, Wyoming, 1468","about":"Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n","registered":"2018-04-26T06:04:40 -02:00","latitude":-64.196802,"longitude":-117.396238,"tags":["wontfix"]} -{"id":50,"isActive":true,"balance":"$1,947.08","picture":"http://placehold.it/32x32","age":21,"color":"Green","name":"Guerra Mcintyre","gender":"male","email":"guerramcintyre@chorizon.com","phone":"+1 (951) 536-2043","address":"423 Lombardy Street, Stewart, West Virginia, 908","about":"Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n","registered":"2015-07-16T05:11:42 -02:00","latitude":79.733743,"longitude":-20.602356,"tags":["bug","good first issue","good first issue"]} -{"id":51,"isActive":true,"balance":"$2,960.90","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Key Cervantes","gender":"male","email":"keycervantes@chorizon.com","phone":"+1 (931) 474-3865","address":"410 Barbey Street, Vernon, Oregon, 2328","about":"Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n","registered":"2019-12-15T12:13:35 -01:00","latitude":47.627647,"longitude":117.049918,"tags":["new issue"]} -{"id":52,"isActive":false,"balance":"$1,884.02","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Karen Nelson","gender":"female","email":"karennelson@chorizon.com","phone":"+1 (993) 528-3607","address":"930 Frank Court, Dunbar, New York, 8810","about":"Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n","registered":"2014-06-23T09:21:44 -02:00","latitude":-59.059033,"longitude":76.565373,"tags":["new issue","bug"]} -{"id":53,"isActive":true,"balance":"$3,559.55","picture":"http://placehold.it/32x32","age":32,"color":"brown","name":"Caitlin Burnett","gender":"female","email":"caitlinburnett@chorizon.com","phone":"+1 (945) 480-2796","address":"516 Senator Street, Emory, Iowa, 4145","about":"In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n","registered":"2019-01-09T02:26:31 -01:00","latitude":-82.774237,"longitude":42.316194,"tags":["bug","good first issue"]} -{"id":54,"isActive":true,"balance":"$2,113.29","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Richards Walls","gender":"male","email":"richardswalls@chorizon.com","phone":"+1 (865) 517-2982","address":"959 Brightwater Avenue, Stevens, Nevada, 2968","about":"Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n","registered":"2014-09-25T06:51:22 -02:00","latitude":80.09202,"longitude":87.49759,"tags":["wontfix","wontfix","bug"]} -{"id":55,"isActive":true,"balance":"$1,977.66","picture":"http://placehold.it/32x32","age":36,"color":"brown","name":"Combs Stanley","gender":"male","email":"combsstanley@chorizon.com","phone":"+1 (827) 419-2053","address":"153 Beverley Road, Siglerville, South Carolina, 3666","about":"Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n","registered":"2019-08-22T07:53:15 -02:00","latitude":78.386181,"longitude":143.661058,"tags":[]} -{"id":56,"isActive":false,"balance":"$3,886.12","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Tucker Barry","gender":"male","email":"tuckerbarry@chorizon.com","phone":"+1 (808) 544-3433","address":"805 Jamaica Avenue, Cornfields, Minnesota, 3689","about":"Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n","registered":"2016-08-29T07:28:00 -02:00","latitude":71.701551,"longitude":9.903068,"tags":[]} -{"id":57,"isActive":false,"balance":"$1,844.56","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Kaitlin Conner","gender":"female","email":"kaitlinconner@chorizon.com","phone":"+1 (862) 467-2666","address":"501 Knight Court, Joppa, Rhode Island, 274","about":"Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n","registered":"2019-05-30T06:38:24 -02:00","latitude":15.613464,"longitude":171.965629,"tags":[]} -{"id":58,"isActive":true,"balance":"$2,876.10","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Mamie Fischer","gender":"female","email":"mamiefischer@chorizon.com","phone":"+1 (948) 545-3901","address":"599 Hunterfly Place, Haena, Georgia, 6005","about":"Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n","registered":"2019-05-27T05:07:10 -02:00","latitude":70.915079,"longitude":-48.813584,"tags":["bug","wontfix","wontfix","good first issue"]} -{"id":59,"isActive":true,"balance":"$1,921.58","picture":"http://placehold.it/32x32","age":31,"color":"Green","name":"Harper Carson","gender":"male","email":"harpercarson@chorizon.com","phone":"+1 (912) 430-3243","address":"883 Dennett Place, Knowlton, New Mexico, 9219","about":"Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n","registered":"2019-12-07T07:33:15 -01:00","latitude":-60.812605,"longitude":-27.129016,"tags":["bug","new issue"]} -{"id":60,"isActive":true,"balance":"$1,770.93","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Jody Herrera","gender":"female","email":"jodyherrera@chorizon.com","phone":"+1 (890) 583-3222","address":"261 Jay Street, Strykersville, Ohio, 9248","about":"Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n","registered":"2016-05-21T01:00:02 -02:00","latitude":-36.846586,"longitude":131.156223,"tags":[]} -{"id":61,"isActive":false,"balance":"$2,813.41","picture":"http://placehold.it/32x32","age":37,"color":"Green","name":"Charles Castillo","gender":"male","email":"charlescastillo@chorizon.com","phone":"+1 (934) 467-2108","address":"675 Morton Street, Rew, Pennsylvania, 137","about":"Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n","registered":"2019-06-10T02:54:22 -02:00","latitude":-16.423202,"longitude":-146.293752,"tags":["new issue","new issue"]} -{"id":62,"isActive":true,"balance":"$3,341.35","picture":"http://placehold.it/32x32","age":33,"color":"blue","name":"Estelle Ramirez","gender":"female","email":"estelleramirez@chorizon.com","phone":"+1 (816) 459-2073","address":"636 Nolans Lane, Camptown, California, 7794","about":"Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n","registered":"2015-02-14T01:05:50 -01:00","latitude":-46.591249,"longitude":-83.385587,"tags":["good first issue","bug"]} -{"id":63,"isActive":true,"balance":"$2,478.30","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Knowles Hebert","gender":"male","email":"knowleshebert@chorizon.com","phone":"+1 (819) 409-2308","address":"361 Kathleen Court, Gratton, Connecticut, 7254","about":"Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n","registered":"2016-03-08T08:34:52 -01:00","latitude":71.042482,"longitude":152.460406,"tags":["good first issue","wontfix"]} -{"id":64,"isActive":false,"balance":"$2,559.09","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Thelma Mckenzie","gender":"female","email":"thelmamckenzie@chorizon.com","phone":"+1 (941) 596-2777","address":"202 Leonard Street, Riverton, Illinois, 8577","about":"Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n","registered":"2020-04-14T12:43:06 -02:00","latitude":16.026129,"longitude":105.464476,"tags":[]} -{"id":65,"isActive":true,"balance":"$1,025.08","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Carole Rowland","gender":"female","email":"carolerowland@chorizon.com","phone":"+1 (862) 558-3448","address":"941 Melba Court, Bluetown, Florida, 9555","about":"Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n","registered":"2014-12-01T05:55:35 -01:00","latitude":-0.191998,"longitude":43.389652,"tags":["wontfix"]} -{"id":66,"isActive":true,"balance":"$1,061.49","picture":"http://placehold.it/32x32","age":35,"color":"brown","name":"Higgins Aguilar","gender":"male","email":"higginsaguilar@chorizon.com","phone":"+1 (911) 540-3791","address":"132 Sackman Street, Layhill, Guam, 8729","about":"Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n","registered":"2015-04-05T02:10:07 -02:00","latitude":74.702813,"longitude":151.314972,"tags":["bug"]} -{"id":67,"isActive":true,"balance":"$3,510.14","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Ilene Gillespie","gender":"female","email":"ilenegillespie@chorizon.com","phone":"+1 (937) 575-2676","address":"835 Lake Street, Naomi, Alabama, 4131","about":"Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n","registered":"2015-06-28T09:41:45 -02:00","latitude":71.573342,"longitude":-95.295989,"tags":["wontfix","wontfix"]} -{"id":68,"isActive":false,"balance":"$1,539.98","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Angelina Dyer","gender":"female","email":"angelinadyer@chorizon.com","phone":"+1 (948) 574-3949","address":"575 Division Place, Gorham, Louisiana, 3458","about":"Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n","registered":"2014-07-08T06:34:36 -02:00","latitude":-85.649593,"longitude":66.126018,"tags":["good first issue"]} -{"id":69,"isActive":true,"balance":"$3,367.69","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Marks Burt","gender":"male","email":"marksburt@chorizon.com","phone":"+1 (895) 497-3138","address":"819 Village Road, Wadsworth, Delaware, 6099","about":"Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n","registered":"2014-08-31T06:12:18 -02:00","latitude":26.854112,"longitude":-143.313948,"tags":["good first issue"]} -{"id":70,"isActive":false,"balance":"$3,755.72","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Glass Perkins","gender":"male","email":"glassperkins@chorizon.com","phone":"+1 (923) 486-3725","address":"899 Roosevelt Court, Belleview, Idaho, 1737","about":"Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n","registered":"2015-05-22T05:44:33 -02:00","latitude":54.27147,"longitude":-65.065604,"tags":["wontfix"]} -{"id":71,"isActive":true,"balance":"$3,381.63","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Candace Sawyer","gender":"female","email":"candacesawyer@chorizon.com","phone":"+1 (830) 404-2636","address":"334 Arkansas Drive, Bordelonville, Tennessee, 8449","about":"Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n","registered":"2014-04-04T08:45:00 -02:00","latitude":6.484262,"longitude":-37.054928,"tags":["new issue","new issue"]} -{"id":72,"isActive":true,"balance":"$1,640.98","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Hendricks Martinez","gender":"male","email":"hendricksmartinez@chorizon.com","phone":"+1 (857) 566-3245","address":"636 Agate Court, Newry, Utah, 3304","about":"Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n","registered":"2018-06-15T10:36:11 -02:00","latitude":86.746034,"longitude":10.347893,"tags":["new issue"]} -{"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]} -{"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]} -{"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]} -{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]} \ No newline at end of file diff --git a/tests/assets/dumps/v1/test/settings.json b/tests/assets/dumps/v1/test/settings.json deleted file mode 100644 index 918cfab53..000000000 --- a/tests/assets/dumps/v1/test/settings.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": "email", - "searchableAttributes": [ - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "displayedAttributes": [ - "id", - "isActive", - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "stopWords": [ - "in", - "ad" - ], - "synonyms": { - "wolverine": ["xmen", "logan"], - "logan": ["wolverine", "xmen"] - }, - "attributesForFaceting": [ - "gender", - "color", - "tags", - "isActive" - ] -} diff --git a/tests/assets/dumps/v1/test/updates.jsonl b/tests/assets/dumps/v1/test/updates.jsonl deleted file mode 100644 index 0dcffdce0..000000000 --- a/tests/assets/dumps/v1/test/updates.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}} -{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} \ No newline at end of file diff --git a/tests/assets/test_set.json b/tests/assets/test_set.json deleted file mode 100644 index 63534c896..000000000 --- a/tests/assets/test_set.json +++ /dev/null @@ -1,1613 +0,0 @@ -[ - { - "id": 0, - "isActive": false, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ] - }, - { - "id": 1, - "isActive": true, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ] - }, - { - "id": 2, - "isActive": true, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ] - }, - { - "id": 3, - "isActive": true, - "balance": "$3,344.40", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Adeline Flynn", - "gender": "female", - "email": "adelineflynn@chorizon.com", - "phone": "+1 (994) 600-2840", - "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948", - "about": "Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n", - "registered": "2014-03-27T06:24:45 -01:00", - "latitude": -74.485173, - "longitude": -11.059859, - "tags": [ - "bug", - "good first issue", - "wontfix", - "new issue" - ] - }, - { - "id": 4, - "isActive": false, - "balance": "$2,575.78", - "picture": "http://placehold.it/32x32", - "age": 39, - "color": "Green", - "name": "Mariana Pacheco", - "gender": "female", - "email": "marianapacheco@chorizon.com", - "phone": "+1 (820) 414-2223", - "address": "664 Rapelye Street, Faywood, California, 7320", - "about": "Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n", - "registered": "2015-09-02T03:23:35 -02:00", - "latitude": 75.763501, - "longitude": -78.777124, - "tags": [ - "new issue" - ] - }, - { - "id": 5, - "isActive": true, - "balance": "$3,793.09", - "picture": "http://placehold.it/32x32", - "age": 20, - "color": "Green", - "name": "Warren Watson", - "gender": "male", - "email": "warrenwatson@chorizon.com", - "phone": "+1 (807) 583-2427", - "address": "671 Prince Street, Faxon, Connecticut, 4275", - "about": "Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n", - "registered": "2017-06-04T06:02:17 -02:00", - "latitude": 29.979223, - "longitude": 25.358943, - "tags": [ - "wontfix", - "wontfix", - "wontfix" - ] - }, - { - "id": 6, - "isActive": true, - "balance": "$2,919.70", - "picture": "http://placehold.it/32x32", - "age": 20, - "color": "blue", - "name": "Shelia Berry", - "gender": "female", - "email": "sheliaberry@chorizon.com", - "phone": "+1 (853) 511-2651", - "address": "437 Forrest Street, Coventry, Illinois, 2056", - "about": "Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n", - "registered": "2018-07-11T02:45:01 -02:00", - "latitude": 54.815991, - "longitude": -118.690609, - "tags": [ - "good first issue", - "bug", - "wontfix", - "new issue" - ] - }, - { - "id": 7, - "isActive": true, - "balance": "$1,349.50", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Chrystal Boyd", - "gender": "female", - "email": "chrystalboyd@chorizon.com", - "phone": "+1 (936) 563-2802", - "address": "670 Croton Loop, Sussex, Florida, 4692", - "about": "Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n", - "registered": "2016-11-01T07:36:04 -01:00", - "latitude": -24.711933, - "longitude": 147.246705, - "tags": [] - }, - { - "id": 8, - "isActive": false, - "balance": "$3,999.56", - "picture": "http://placehold.it/32x32", - "age": 30, - "color": "brown", - "name": "Martin Porter", - "gender": "male", - "email": "martinporter@chorizon.com", - "phone": "+1 (895) 580-2304", - "address": "577 Regent Place, Aguila, Guam, 6554", - "about": "Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n", - "registered": "2014-09-20T02:08:30 -02:00", - "latitude": -88.344273, - "longitude": 37.964466, - "tags": [] - }, - { - "id": 9, - "isActive": true, - "balance": "$3,729.71", - "picture": "http://placehold.it/32x32", - "age": 26, - "color": "blue", - "name": "Kelli Mendez", - "gender": "female", - "email": "kellimendez@chorizon.com", - "phone": "+1 (936) 401-2236", - "address": "242 Caton Place, Grazierville, Alabama, 3968", - "about": "Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n", - "registered": "2018-05-04T10:35:30 -02:00", - "latitude": 49.37551, - "longitude": 41.872323, - "tags": [ - "new issue", - "new issue" - ] - }, - { - "id": 10, - "isActive": false, - "balance": "$1,127.47", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "blue", - "name": "Maddox Johns", - "gender": "male", - "email": "maddoxjohns@chorizon.com", - "phone": "+1 (892) 470-2357", - "address": "756 Beard Street, Avalon, Louisiana, 114", - "about": "Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n", - "registered": "2016-04-22T06:41:25 -02:00", - "latitude": 66.640229, - "longitude": -17.222666, - "tags": [ - "new issue", - "good first issue", - "good first issue", - "new issue" - ] - }, - { - "id": 11, - "isActive": true, - "balance": "$1,351.43", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Evans Wagner", - "gender": "male", - "email": "evanswagner@chorizon.com", - "phone": "+1 (889) 496-2332", - "address": "118 Monaco Place, Lutsen, Delaware, 6209", - "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", - "registered": "2016-10-27T01:26:31 -02:00", - "latitude": -77.673222, - "longitude": -142.657214, - "tags": [ - "good first issue", - "good first issue" - ] - }, - { - "id": 12, - "isActive": false, - "balance": "$3,394.96", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "blue", - "name": "Aida Kirby", - "gender": "female", - "email": "aidakirby@chorizon.com", - "phone": "+1 (942) 532-2325", - "address": "797 Engert Avenue, Wilsonia, Idaho, 6532", - "about": "Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n", - "registered": "2018-06-18T04:39:57 -02:00", - "latitude": -58.062041, - "longitude": 34.999254, - "tags": [ - "new issue", - "wontfix", - "bug", - "new issue" - ] - }, - { - "id": 13, - "isActive": true, - "balance": "$2,812.62", - "picture": "http://placehold.it/32x32", - "age": 40, - "color": "blue", - "name": "Nelda Burris", - "gender": "female", - "email": "neldaburris@chorizon.com", - "phone": "+1 (813) 600-2576", - "address": "160 Opal Court, Fowlerville, Tennessee, 2170", - "about": "Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n", - "registered": "2015-08-15T12:39:53 -02:00", - "latitude": 66.6871, - "longitude": 179.549488, - "tags": [ - "wontfix" - ] - }, - { - "id": 14, - "isActive": true, - "balance": "$1,718.33", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Jennifer Hart", - "gender": "female", - "email": "jenniferhart@chorizon.com", - "phone": "+1 (850) 537-2513", - "address": "124 Veranda Place, Nash, Utah, 985", - "about": "Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n", - "registered": "2016-09-04T11:46:59 -02:00", - "latitude": -66.827751, - "longitude": 99.220079, - "tags": [ - "wontfix", - "bug", - "new issue", - "new issue" - ] - }, - { - "id": 15, - "isActive": false, - "balance": "$2,698.16", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "blue", - "name": "Aurelia Contreras", - "gender": "female", - "email": "aureliacontreras@chorizon.com", - "phone": "+1 (932) 442-3103", - "address": "655 Dwight Street, Grapeview, Palau, 8356", - "about": "Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n", - "registered": "2014-09-11T10:43:15 -02:00", - "latitude": -71.328973, - "longitude": 133.404895, - "tags": [ - "wontfix", - "bug", - "good first issue" - ] - }, - { - "id": 16, - "isActive": true, - "balance": "$3,303.25", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Estella Bass", - "gender": "female", - "email": "estellabass@chorizon.com", - "phone": "+1 (825) 436-2909", - "address": "435 Rockwell Place, Garberville, Wisconsin, 2230", - "about": "Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n", - "registered": "2017-11-23T09:32:09 -01:00", - "latitude": 81.17014, - "longitude": -145.262693, - "tags": [ - "new issue" - ] - }, - { - "id": 17, - "isActive": false, - "balance": "$3,579.20", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "brown", - "name": "Ortega Brennan", - "gender": "male", - "email": "ortegabrennan@chorizon.com", - "phone": "+1 (906) 526-2287", - "address": "440 Berry Street, Rivera, Maine, 1849", - "about": "Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n", - "registered": "2016-03-31T02:17:13 -02:00", - "latitude": -68.407524, - "longitude": -113.642067, - "tags": [ - "new issue", - "wontfix" - ] - }, - { - "id": 18, - "isActive": false, - "balance": "$1,484.92", - "picture": "http://placehold.it/32x32", - "age": 39, - "color": "blue", - "name": "Leonard Tillman", - "gender": "male", - "email": "leonardtillman@chorizon.com", - "phone": "+1 (864) 541-3456", - "address": "985 Provost Street, Charco, New Hampshire, 8632", - "about": "Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n", - "registered": "2018-05-06T08:21:27 -02:00", - "latitude": -8.581801, - "longitude": -61.910062, - "tags": [ - "wontfix", - "new issue", - "bug", - "bug" - ] - }, - { - "id": 19, - "isActive": true, - "balance": "$3,572.55", - "picture": "http://placehold.it/32x32", - "age": 33, - "color": "brown", - "name": "Dale Payne", - "gender": "male", - "email": "dalepayne@chorizon.com", - "phone": "+1 (814) 469-3499", - "address": "536 Dare Court, Ironton, Arkansas, 8605", - "about": "Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n", - "registered": "2019-10-11T01:01:33 -02:00", - "latitude": -18.280968, - "longitude": -126.091797, - "tags": [ - "bug", - "wontfix", - "wontfix", - "wontfix" - ] - }, - { - "id": 20, - "isActive": true, - "balance": "$1,986.48", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Florence Long", - "gender": "female", - "email": "florencelong@chorizon.com", - "phone": "+1 (972) 557-3858", - "address": "519 Hendrickson Street, Templeton, Hawaii, 2389", - "about": "Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n", - "registered": "2016-05-02T09:18:59 -02:00", - "latitude": -27.110866, - "longitude": -45.09445, - "tags": [] - }, - { - "id": 21, - "isActive": true, - "balance": "$1,440.09", - "picture": "http://placehold.it/32x32", - "age": 40, - "color": "blue", - "name": "Levy Whitley", - "gender": "male", - "email": "levywhitley@chorizon.com", - "phone": "+1 (911) 458-2411", - "address": "187 Thomas Street, Hachita, North Carolina, 2989", - "about": "Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n", - "registered": "2014-04-30T07:31:38 -02:00", - "latitude": -6.537315, - "longitude": 171.813536, - "tags": [ - "bug" - ] - }, - { - "id": 22, - "isActive": false, - "balance": "$2,938.57", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Bernard Mcfarland", - "gender": "male", - "email": "bernardmcfarland@chorizon.com", - "phone": "+1 (979) 442-3386", - "address": "409 Hall Street, Keyport, Federated States Of Micronesia, 7011", - "about": "Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n", - "registered": "2017-08-10T10:07:59 -02:00", - "latitude": 63.766795, - "longitude": 68.177069, - "tags": [] - }, - { - "id": 23, - "isActive": true, - "balance": "$1,678.49", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "brown", - "name": "Blanca Mcclain", - "gender": "female", - "email": "blancamcclain@chorizon.com", - "phone": "+1 (976) 439-2772", - "address": "176 Crooke Avenue, Valle, Virginia, 5373", - "about": "Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n", - "registered": "2015-10-12T11:57:28 -02:00", - "latitude": -8.944564, - "longitude": -150.711709, - "tags": [ - "bug", - "wontfix", - "good first issue" - ] - }, - { - "id": 24, - "isActive": true, - "balance": "$2,276.87", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Espinoza Ford", - "gender": "male", - "email": "espinozaford@chorizon.com", - "phone": "+1 (945) 429-3975", - "address": "137 Bowery Street, Itmann, District Of Columbia, 1864", - "about": "Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n", - "registered": "2014-03-26T02:16:08 -01:00", - "latitude": -37.137666, - "longitude": -51.811757, - "tags": [ - "wontfix", - "bug" - ] - }, - { - "id": 25, - "isActive": true, - "balance": "$3,973.43", - "picture": "http://placehold.it/32x32", - "age": 29, - "color": "Green", - "name": "Sykes Conley", - "gender": "male", - "email": "sykesconley@chorizon.com", - "phone": "+1 (851) 401-3916", - "address": "345 Grand Street, Woodlands, Missouri, 4461", - "about": "Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n", - "registered": "2015-09-12T06:03:56 -02:00", - "latitude": 67.282955, - "longitude": -64.341323, - "tags": [ - "wontfix" - ] - }, - { - "id": 26, - "isActive": false, - "balance": "$1,431.50", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Barlow Duran", - "gender": "male", - "email": "barlowduran@chorizon.com", - "phone": "+1 (995) 436-2562", - "address": "481 Everett Avenue, Allison, Nebraska, 3065", - "about": "Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n", - "registered": "2017-06-29T04:28:43 -02:00", - "latitude": -38.70606, - "longitude": 55.02816, - "tags": [ - "new issue" - ] - }, - { - "id": 27, - "isActive": true, - "balance": "$3,478.27", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "blue", - "name": "Schwartz Morgan", - "gender": "male", - "email": "schwartzmorgan@chorizon.com", - "phone": "+1 (861) 507-2067", - "address": "451 Lincoln Road, Fairlee, Washington, 2717", - "about": "Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n", - "registered": "2016-05-10T08:34:54 -02:00", - "latitude": -75.886403, - "longitude": 93.044471, - "tags": [ - "bug", - "bug", - "wontfix", - "wontfix" - ] - }, - { - "id": 28, - "isActive": true, - "balance": "$2,825.59", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Kristy Leon", - "gender": "female", - "email": "kristyleon@chorizon.com", - "phone": "+1 (948) 465-2563", - "address": "594 Macon Street, Floris, South Dakota, 3565", - "about": "Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n", - "registered": "2014-12-14T04:10:29 -01:00", - "latitude": -50.01615, - "longitude": -68.908804, - "tags": [ - "wontfix", - "good first issue" - ] - }, - { - "id": 29, - "isActive": false, - "balance": "$3,028.03", - "picture": "http://placehold.it/32x32", - "age": 39, - "color": "blue", - "name": "Ashley Pittman", - "gender": "male", - "email": "ashleypittman@chorizon.com", - "phone": "+1 (928) 507-3523", - "address": "646 Adelphi Street, Clara, Colorado, 6056", - "about": "Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n", - "registered": "2016-01-07T10:40:48 -01:00", - "latitude": -58.766037, - "longitude": -124.828485, - "tags": [ - "wontfix" - ] - }, - { - "id": 30, - "isActive": true, - "balance": "$2,021.11", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Stacy Espinoza", - "gender": "female", - "email": "stacyespinoza@chorizon.com", - "phone": "+1 (999) 487-3253", - "address": "931 Alabama Avenue, Bangor, Alaska, 8215", - "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", - "registered": "2014-07-16T06:15:53 -02:00", - "latitude": 41.560197, - "longitude": 177.697, - "tags": [ - "new issue", - "new issue", - "bug" - ] - }, - { - "id": 31, - "isActive": false, - "balance": "$3,609.82", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Vilma Garza", - "gender": "female", - "email": "vilmagarza@chorizon.com", - "phone": "+1 (944) 585-2021", - "address": "565 Tech Place, Sedley, Puerto Rico, 858", - "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", - "registered": "2017-06-30T07:43:52 -02:00", - "latitude": -12.574889, - "longitude": -54.771186, - "tags": [ - "new issue", - "wontfix", - "wontfix" - ] - }, - { - "id": 32, - "isActive": false, - "balance": "$2,882.34", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "brown", - "name": "June Dunlap", - "gender": "female", - "email": "junedunlap@chorizon.com", - "phone": "+1 (997) 504-2937", - "address": "353 Cozine Avenue, Goodville, Indiana, 1438", - "about": "Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n", - "registered": "2016-08-23T08:54:11 -02:00", - "latitude": -27.883363, - "longitude": -163.919683, - "tags": [ - "new issue", - "new issue", - "bug", - "wontfix" - ] - }, - { - "id": 33, - "isActive": true, - "balance": "$3,556.54", - "picture": "http://placehold.it/32x32", - "age": 33, - "color": "brown", - "name": "Cecilia Greer", - "gender": "female", - "email": "ceciliagreer@chorizon.com", - "phone": "+1 (977) 573-3498", - "address": "696 Withers Street, Lydia, Oklahoma, 3220", - "about": "Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n", - "registered": "2017-01-13T11:30:12 -01:00", - "latitude": 60.467215, - "longitude": 84.684575, - "tags": [ - "wontfix", - "good first issue", - "good first issue", - "wontfix" - ] - }, - { - "id": 34, - "isActive": true, - "balance": "$1,413.35", - "picture": "http://placehold.it/32x32", - "age": 33, - "color": "brown", - "name": "Mckay Schroeder", - "gender": "male", - "email": "mckayschroeder@chorizon.com", - "phone": "+1 (816) 480-3657", - "address": "958 Miami Court, Rehrersburg, Northern Mariana Islands, 567", - "about": "Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n", - "registered": "2016-02-08T04:50:15 -01:00", - "latitude": -72.413287, - "longitude": -159.254371, - "tags": [ - "good first issue" - ] - }, - { - "id": 35, - "isActive": true, - "balance": "$2,306.53", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Sawyer Mccormick", - "gender": "male", - "email": "sawyermccormick@chorizon.com", - "phone": "+1 (829) 569-3012", - "address": "749 Apollo Street, Eastvale, Texas, 7373", - "about": "Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n", - "registered": "2019-11-30T11:53:23 -01:00", - "latitude": -48.978194, - "longitude": 110.950191, - "tags": [ - "good first issue", - "new issue", - "new issue", - "bug" - ] - }, - { - "id": 36, - "isActive": false, - "balance": "$1,844.54", - "picture": "http://placehold.it/32x32", - "age": 37, - "color": "brown", - "name": "Barbra Valenzuela", - "gender": "female", - "email": "barbravalenzuela@chorizon.com", - "phone": "+1 (992) 512-2649", - "address": "617 Schenck Court, Reinerton, Michigan, 2908", - "about": "Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n", - "registered": "2019-03-29T01:59:31 -01:00", - "latitude": 45.193723, - "longitude": -12.486778, - "tags": [ - "new issue", - "new issue", - "wontfix", - "wontfix" - ] - }, - { - "id": 37, - "isActive": false, - "balance": "$3,469.82", - "picture": "http://placehold.it/32x32", - "age": 39, - "color": "brown", - "name": "Opal Weiss", - "gender": "female", - "email": "opalweiss@chorizon.com", - "phone": "+1 (809) 400-3079", - "address": "535 Bogart Street, Frizzleburg, Arizona, 5222", - "about": "Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n", - "registered": "2019-09-04T07:22:28 -02:00", - "latitude": 72.50376, - "longitude": 61.656435, - "tags": [ - "bug", - "bug", - "good first issue", - "good first issue" - ] - }, - { - "id": 38, - "isActive": true, - "balance": "$1,992.38", - "picture": "http://placehold.it/32x32", - "age": 40, - "color": "Green", - "name": "Christina Short", - "gender": "female", - "email": "christinashort@chorizon.com", - "phone": "+1 (884) 589-2705", - "address": "594 Willmohr Street, Dexter, Montana, 660", - "about": "Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n", - "registered": "2014-01-21T09:31:56 -01:00", - "latitude": -42.762739, - "longitude": 77.052349, - "tags": [ - "bug", - "new issue" - ] - }, - { - "id": 39, - "isActive": false, - "balance": "$1,722.85", - "picture": "http://placehold.it/32x32", - "age": 29, - "color": "brown", - "name": "Golden Horton", - "gender": "male", - "email": "goldenhorton@chorizon.com", - "phone": "+1 (903) 426-2489", - "address": "191 Schenck Avenue, Mayfair, North Dakota, 5000", - "about": "Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n", - "registered": "2015-08-19T02:56:41 -02:00", - "latitude": 69.922534, - "longitude": 9.881433, - "tags": [ - "bug" - ] - }, - { - "id": 40, - "isActive": false, - "balance": "$1,656.54", - "picture": "http://placehold.it/32x32", - "age": 21, - "color": "blue", - "name": "Stafford Emerson", - "gender": "male", - "email": "staffordemerson@chorizon.com", - "phone": "+1 (992) 455-2573", - "address": "523 Thornton Street, Conway, Vermont, 6331", - "about": "Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n", - "registered": "2019-02-16T04:07:08 -01:00", - "latitude": -29.143111, - "longitude": -57.207703, - "tags": [ - "wontfix", - "good first issue", - "good first issue" - ] - }, - { - "id": 41, - "isActive": false, - "balance": "$1,861.56", - "picture": "http://placehold.it/32x32", - "age": 21, - "color": "brown", - "name": "Salinas Gamble", - "gender": "male", - "email": "salinasgamble@chorizon.com", - "phone": "+1 (901) 525-2373", - "address": "991 Nostrand Avenue, Kansas, Mississippi, 6756", - "about": "Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n", - "registered": "2017-08-21T05:47:53 -02:00", - "latitude": -22.593819, - "longitude": -63.613004, - "tags": [ - "good first issue", - "bug", - "bug", - "wontfix" - ] - }, - { - "id": 42, - "isActive": true, - "balance": "$3,179.74", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "brown", - "name": "Graciela Russell", - "gender": "female", - "email": "gracielarussell@chorizon.com", - "phone": "+1 (893) 464-3951", - "address": "361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713", - "about": "Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n", - "registered": "2015-05-18T09:52:56 -02:00", - "latitude": -14.634444, - "longitude": 12.931783, - "tags": [ - "wontfix", - "bug", - "wontfix" - ] - }, - { - "id": 43, - "isActive": true, - "balance": "$1,777.38", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "blue", - "name": "Arnold Bender", - "gender": "male", - "email": "arnoldbender@chorizon.com", - "phone": "+1 (945) 581-3808", - "address": "781 Lorraine Street, Gallina, American Samoa, 1832", - "about": "Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n", - "registered": "2018-12-23T02:26:30 -01:00", - "latitude": 41.208579, - "longitude": 51.948925, - "tags": [ - "bug", - "good first issue", - "good first issue", - "wontfix" - ] - }, - { - "id": 44, - "isActive": true, - "balance": "$2,893.45", - "picture": "http://placehold.it/32x32", - "age": 22, - "color": "Green", - "name": "Joni Spears", - "gender": "female", - "email": "jonispears@chorizon.com", - "phone": "+1 (916) 565-2124", - "address": "307 Harwood Place, Canterwood, Maryland, 2047", - "about": "Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n", - "registered": "2015-03-01T12:38:28 -01:00", - "latitude": 8.19071, - "longitude": 146.323808, - "tags": [ - "wontfix", - "new issue", - "good first issue", - "good first issue" - ] - }, - { - "id": 45, - "isActive": true, - "balance": "$2,830.36", - "picture": "http://placehold.it/32x32", - "age": 20, - "color": "brown", - "name": "Irene Bennett", - "gender": "female", - "email": "irenebennett@chorizon.com", - "phone": "+1 (904) 431-2211", - "address": "353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686", - "about": "Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n", - "registered": "2018-04-17T05:18:51 -02:00", - "latitude": -36.435177, - "longitude": -127.552573, - "tags": [ - "bug", - "wontfix" - ] - }, - { - "id": 46, - "isActive": true, - "balance": "$1,348.04", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "Green", - "name": "Lawson Curtis", - "gender": "male", - "email": "lawsoncurtis@chorizon.com", - "phone": "+1 (896) 532-2172", - "address": "942 Gerritsen Avenue, Southmont, Kansas, 8915", - "about": "Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n", - "registered": "2016-08-23T01:41:09 -02:00", - "latitude": -48.783539, - "longitude": 20.492944, - "tags": [] - }, - { - "id": 47, - "isActive": true, - "balance": "$1,132.41", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Goff May", - "gender": "male", - "email": "goffmay@chorizon.com", - "phone": "+1 (859) 453-3415", - "address": "225 Rutledge Street, Boonville, Massachusetts, 4081", - "about": "Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n", - "registered": "2014-10-25T07:32:30 -02:00", - "latitude": 13.079225, - "longitude": 76.215086, - "tags": [ - "bug" - ] - }, - { - "id": 48, - "isActive": true, - "balance": "$1,201.87", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Goodman Becker", - "gender": "male", - "email": "goodmanbecker@chorizon.com", - "phone": "+1 (825) 470-3437", - "address": "388 Seigel Street, Sisquoc, Kentucky, 8231", - "about": "Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n", - "registered": "2019-09-05T04:49:03 -02:00", - "latitude": -23.792094, - "longitude": -13.621221, - "tags": [ - "bug", - "bug", - "wontfix", - "new issue" - ] - }, - { - "id": 49, - "isActive": true, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ] - }, - { - "id": 50, - "isActive": true, - "balance": "$1,947.08", - "picture": "http://placehold.it/32x32", - "age": 21, - "color": "Green", - "name": "Guerra Mcintyre", - "gender": "male", - "email": "guerramcintyre@chorizon.com", - "phone": "+1 (951) 536-2043", - "address": "423 Lombardy Street, Stewart, West Virginia, 908", - "about": "Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n", - "registered": "2015-07-16T05:11:42 -02:00", - "latitude": 79.733743, - "longitude": -20.602356, - "tags": [ - "bug", - "good first issue", - "good first issue" - ] - }, - { - "id": 51, - "isActive": true, - "balance": "$2,960.90", - "picture": "http://placehold.it/32x32", - "age": 23, - "color": "blue", - "name": "Key Cervantes", - "gender": "male", - "email": "keycervantes@chorizon.com", - "phone": "+1 (931) 474-3865", - "address": "410 Barbey Street, Vernon, Oregon, 2328", - "about": "Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n", - "registered": "2019-12-15T12:13:35 -01:00", - "latitude": 47.627647, - "longitude": 117.049918, - "tags": [ - "new issue" - ] - }, - { - "id": 52, - "isActive": false, - "balance": "$1,884.02", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Karen Nelson", - "gender": "female", - "email": "karennelson@chorizon.com", - "phone": "+1 (993) 528-3607", - "address": "930 Frank Court, Dunbar, New York, 8810", - "about": "Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n", - "registered": "2014-06-23T09:21:44 -02:00", - "latitude": -59.059033, - "longitude": 76.565373, - "tags": [ - "new issue", - "bug" - ] - }, - { - "id": 53, - "isActive": true, - "balance": "$3,559.55", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "brown", - "name": "Caitlin Burnett", - "gender": "female", - "email": "caitlinburnett@chorizon.com", - "phone": "+1 (945) 480-2796", - "address": "516 Senator Street, Emory, Iowa, 4145", - "about": "In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n", - "registered": "2019-01-09T02:26:31 -01:00", - "latitude": -82.774237, - "longitude": 42.316194, - "tags": [ - "bug", - "good first issue" - ] - }, - { - "id": 54, - "isActive": true, - "balance": "$2,113.29", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Richards Walls", - "gender": "male", - "email": "richardswalls@chorizon.com", - "phone": "+1 (865) 517-2982", - "address": "959 Brightwater Avenue, Stevens, Nevada, 2968", - "about": "Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n", - "registered": "2014-09-25T06:51:22 -02:00", - "latitude": 80.09202, - "longitude": 87.49759, - "tags": [ - "wontfix", - "wontfix", - "bug" - ] - }, - { - "id": 55, - "isActive": true, - "balance": "$1,977.66", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "brown", - "name": "Combs Stanley", - "gender": "male", - "email": "combsstanley@chorizon.com", - "phone": "+1 (827) 419-2053", - "address": "153 Beverley Road, Siglerville, South Carolina, 3666", - "about": "Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n", - "registered": "2019-08-22T07:53:15 -02:00", - "latitude": 78.386181, - "longitude": 143.661058, - "tags": [] - }, - { - "id": 56, - "isActive": false, - "balance": "$3,886.12", - "picture": "http://placehold.it/32x32", - "age": 23, - "color": "brown", - "name": "Tucker Barry", - "gender": "male", - "email": "tuckerbarry@chorizon.com", - "phone": "+1 (808) 544-3433", - "address": "805 Jamaica Avenue, Cornfields, Minnesota, 3689", - "about": "Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n", - "registered": "2016-08-29T07:28:00 -02:00", - "latitude": 71.701551, - "longitude": 9.903068, - "tags": [] - }, - { - "id": 57, - "isActive": false, - "balance": "$1,844.56", - "picture": "http://placehold.it/32x32", - "age": 20, - "color": "Green", - "name": "Kaitlin Conner", - "gender": "female", - "email": "kaitlinconner@chorizon.com", - "phone": "+1 (862) 467-2666", - "address": "501 Knight Court, Joppa, Rhode Island, 274", - "about": "Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n", - "registered": "2019-05-30T06:38:24 -02:00", - "latitude": 15.613464, - "longitude": 171.965629, - "tags": [] - }, - { - "id": 58, - "isActive": true, - "balance": "$2,876.10", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Mamie Fischer", - "gender": "female", - "email": "mamiefischer@chorizon.com", - "phone": "+1 (948) 545-3901", - "address": "599 Hunterfly Place, Haena, Georgia, 6005", - "about": "Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n", - "registered": "2019-05-27T05:07:10 -02:00", - "latitude": 70.915079, - "longitude": -48.813584, - "tags": [ - "bug", - "wontfix", - "wontfix", - "good first issue" - ] - }, - { - "id": 59, - "isActive": true, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ] - }, - { - "id": 60, - "isActive": true, - "balance": "$1,770.93", - "picture": "http://placehold.it/32x32", - "age": 23, - "color": "brown", - "name": "Jody Herrera", - "gender": "female", - "email": "jodyherrera@chorizon.com", - "phone": "+1 (890) 583-3222", - "address": "261 Jay Street, Strykersville, Ohio, 9248", - "about": "Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n", - "registered": "2016-05-21T01:00:02 -02:00", - "latitude": -36.846586, - "longitude": 131.156223, - "tags": [] - }, - { - "id": 61, - "isActive": false, - "balance": "$2,813.41", - "picture": "http://placehold.it/32x32", - "age": 37, - "color": "Green", - "name": "Charles Castillo", - "gender": "male", - "email": "charlescastillo@chorizon.com", - "phone": "+1 (934) 467-2108", - "address": "675 Morton Street, Rew, Pennsylvania, 137", - "about": "Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n", - "registered": "2019-06-10T02:54:22 -02:00", - "latitude": -16.423202, - "longitude": -146.293752, - "tags": [ - "new issue", - "new issue" - ] - }, - { - "id": 62, - "isActive": true, - "balance": "$3,341.35", - "picture": "http://placehold.it/32x32", - "age": 33, - "color": "blue", - "name": "Estelle Ramirez", - "gender": "female", - "email": "estelleramirez@chorizon.com", - "phone": "+1 (816) 459-2073", - "address": "636 Nolans Lane, Camptown, California, 7794", - "about": "Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n", - "registered": "2015-02-14T01:05:50 -01:00", - "latitude": -46.591249, - "longitude": -83.385587, - "tags": [ - "good first issue", - "bug" - ] - }, - { - "id": 63, - "isActive": true, - "balance": "$2,478.30", - "picture": "http://placehold.it/32x32", - "age": 21, - "color": "blue", - "name": "Knowles Hebert", - "gender": "male", - "email": "knowleshebert@chorizon.com", - "phone": "+1 (819) 409-2308", - "address": "361 Kathleen Court, Gratton, Connecticut, 7254", - "about": "Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n", - "registered": "2016-03-08T08:34:52 -01:00", - "latitude": 71.042482, - "longitude": 152.460406, - "tags": [ - "good first issue", - "wontfix" - ] - }, - { - "id": 64, - "isActive": false, - "balance": "$2,559.09", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Thelma Mckenzie", - "gender": "female", - "email": "thelmamckenzie@chorizon.com", - "phone": "+1 (941) 596-2777", - "address": "202 Leonard Street, Riverton, Illinois, 8577", - "about": "Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n", - "registered": "2020-04-14T12:43:06 -02:00", - "latitude": 16.026129, - "longitude": 105.464476, - "tags": [] - }, - { - "id": 65, - "isActive": true, - "balance": "$1,025.08", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Carole Rowland", - "gender": "female", - "email": "carolerowland@chorizon.com", - "phone": "+1 (862) 558-3448", - "address": "941 Melba Court, Bluetown, Florida, 9555", - "about": "Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n", - "registered": "2014-12-01T05:55:35 -01:00", - "latitude": -0.191998, - "longitude": 43.389652, - "tags": [ - "wontfix" - ] - }, - { - "id": 66, - "isActive": true, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ] - }, - { - "id": 67, - "isActive": true, - "balance": "$3,510.14", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Ilene Gillespie", - "gender": "female", - "email": "ilenegillespie@chorizon.com", - "phone": "+1 (937) 575-2676", - "address": "835 Lake Street, Naomi, Alabama, 4131", - "about": "Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n", - "registered": "2015-06-28T09:41:45 -02:00", - "latitude": 71.573342, - "longitude": -95.295989, - "tags": [ - "wontfix", - "wontfix" - ] - }, - { - "id": 68, - "isActive": false, - "balance": "$1,539.98", - "picture": "http://placehold.it/32x32", - "age": 24, - "color": "Green", - "name": "Angelina Dyer", - "gender": "female", - "email": "angelinadyer@chorizon.com", - "phone": "+1 (948) 574-3949", - "address": "575 Division Place, Gorham, Louisiana, 3458", - "about": "Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n", - "registered": "2014-07-08T06:34:36 -02:00", - "latitude": -85.649593, - "longitude": 66.126018, - "tags": [ - "good first issue" - ] - }, - { - "id": 69, - "isActive": true, - "balance": "$3,367.69", - "picture": "http://placehold.it/32x32", - "age": 30, - "color": "brown", - "name": "Marks Burt", - "gender": "male", - "email": "marksburt@chorizon.com", - "phone": "+1 (895) 497-3138", - "address": "819 Village Road, Wadsworth, Delaware, 6099", - "about": "Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n", - "registered": "2014-08-31T06:12:18 -02:00", - "latitude": 26.854112, - "longitude": -143.313948, - "tags": [ - "good first issue" - ] - }, - { - "id": 70, - "isActive": false, - "balance": "$3,755.72", - "picture": "http://placehold.it/32x32", - "age": 23, - "color": "blue", - "name": "Glass Perkins", - "gender": "male", - "email": "glassperkins@chorizon.com", - "phone": "+1 (923) 486-3725", - "address": "899 Roosevelt Court, Belleview, Idaho, 1737", - "about": "Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n", - "registered": "2015-05-22T05:44:33 -02:00", - "latitude": 54.27147, - "longitude": -65.065604, - "tags": [ - "wontfix" - ] - }, - { - "id": 71, - "isActive": true, - "balance": "$3,381.63", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Candace Sawyer", - "gender": "female", - "email": "candacesawyer@chorizon.com", - "phone": "+1 (830) 404-2636", - "address": "334 Arkansas Drive, Bordelonville, Tennessee, 8449", - "about": "Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n", - "registered": "2014-04-04T08:45:00 -02:00", - "latitude": 6.484262, - "longitude": -37.054928, - "tags": [ - "new issue", - "new issue" - ] - }, - { - "id": 72, - "isActive": true, - "balance": "$1,640.98", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Hendricks Martinez", - "gender": "male", - "email": "hendricksmartinez@chorizon.com", - "phone": "+1 (857) 566-3245", - "address": "636 Agate Court, Newry, Utah, 3304", - "about": "Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n", - "registered": "2018-06-15T10:36:11 -02:00", - "latitude": 86.746034, - "longitude": 10.347893, - "tags": [ - "new issue" - ] - }, - { - "id": 73, - "isActive": false, - "balance": "$1,239.74", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "blue", - "name": "Eleanor Shepherd", - "gender": "female", - "email": "eleanorshepherd@chorizon.com", - "phone": "+1 (894) 567-2617", - "address": "670 Lafayette Walk, Darlington, Palau, 8803", - "about": "Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n", - "registered": "2020-02-29T12:15:28 -01:00", - "latitude": 35.749621, - "longitude": -94.40842, - "tags": [ - "good first issue", - "new issue", - "new issue", - "bug" - ] - }, - { - "id": 74, - "isActive": true, - "balance": "$1,180.90", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Stark Wong", - "gender": "male", - "email": "starkwong@chorizon.com", - "phone": "+1 (805) 575-3055", - "address": "522 Bond Street, Bawcomville, Wisconsin, 324", - "about": "Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n", - "registered": "2020-01-25T10:47:48 -01:00", - "latitude": -80.452139, - "longitude": 160.72546, - "tags": [ - "wontfix" - ] - }, - { - "id": 75, - "isActive": false, - "balance": "$1,913.42", - "picture": "http://placehold.it/32x32", - "age": 24, - "color": "Green", - "name": "Emma Jacobs", - "gender": "female", - "email": "emmajacobs@chorizon.com", - "phone": "+1 (899) 554-3847", - "address": "173 Tapscott Street, Esmont, Maine, 7450", - "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", - "registered": "2019-03-29T06:24:13 -01:00", - "latitude": -35.53722, - "longitude": 155.703874, - "tags": [] - }, - { - "id": 76, - "isActive": false, - "balance": "$1,274.29", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "Green", - "name": "Clarice Gardner", - "gender": "female", - "email": "claricegardner@chorizon.com", - "phone": "+1 (810) 407-3258", - "address": "894 Brooklyn Road, Utting, New Hampshire, 6404", - "about": "Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n", - "registered": "2014-10-20T10:13:32 -02:00", - "latitude": 17.11935, - "longitude": 65.38197, - "tags": [ - "new issue", - "wontfix" - ] - } -] diff --git a/tests/common.rs b/tests/common.rs deleted file mode 100644 index 43cea1447..000000000 --- a/tests/common.rs +++ /dev/null @@ -1,569 +0,0 @@ -#![allow(dead_code)] - -use actix_web::{http::StatusCode, test}; -use serde_json::{json, Value}; -use std::time::Duration; -use tempdir::TempDir; -use tokio::time::delay_for; - -use meilisearch_core::DatabaseOptions; -use meilisearch_http::data::Data; -use meilisearch_http::helpers::NormalizePath; -use meilisearch_http::option::Opt; - -/// Performs a search test on both post and get routes -#[macro_export] -macro_rules! test_post_get_search { - ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => { - let post_query: meilisearch_http::routes::search::SearchQueryPost = - serde_json::from_str(&$query.clone().to_string()).unwrap(); - let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into(); - let get_query = ::serde_url_params::to_string(&get_query).unwrap(); - let ($response, $status_code) = $server.search_get(&get_query).await; - let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { - panic!( - "panic in get route: {:?}", - e.downcast_ref::<&str>().unwrap() - ) - }); - let ($response, $status_code) = $server.search_post($query).await; - let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { - panic!( - "panic in post route: {:?}", - e.downcast_ref::<&str>().unwrap() - ) - }); - }; -} - -pub struct Server { - pub uid: String, - pub data: Data, -} - -impl Server { - pub fn with_uid(uid: &str) -> Server { - let tmp_dir = TempDir::new("meilisearch").unwrap(); - - let default_db_options = DatabaseOptions::default(); - - let opt = Opt { - db_path: tmp_dir.path().join("db").to_str().unwrap().to_string(), - dumps_dir: tmp_dir.path().join("dump"), - dump_batch_size: 16, - http_addr: "127.0.0.1:7700".to_owned(), - master_key: None, - env: "development".to_owned(), - no_analytics: true, - max_mdb_size: default_db_options.main_map_size, - max_udb_size: default_db_options.update_map_size, - http_payload_size_limit: 10000000, - ..Opt::default() - }; - - let data = Data::new(opt).unwrap(); - - Server { - uid: uid.to_string(), - data, - } - } - - pub async fn test_server() -> Self { - let mut server = Self::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - ], - "searchableAttributes": [ - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags", - ], - "displayedAttributes": [ - "id", - "isActive", - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags", - ], - }); - - server.update_all_settings(body).await; - - let dataset = include_bytes!("assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - server.add_or_replace_multiple_documents(body).await; - server - } - - pub fn data(&self) -> &Data { - &self.data - } - - pub async fn wait_update_id(&mut self, update_id: u64) { - // try 10 times to get status, or panic to not wait forever - for _ in 0..10 { - let (response, status_code) = self.get_update_status(update_id).await; - assert_eq!(status_code, 200); - - if response["status"] == "processed" || response["status"] == "failed" { - // eprintln!("{:#?}", response); - return; - } - - delay_for(Duration::from_secs(1)).await; - } - panic!("Timeout waiting for update id"); - } - - // Global Http request GET/POST/DELETE async or sync - - pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("get_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::get().uri(url).to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn post_request(&self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("post_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::post() - .uri(url) - .set_json(&body) - .to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("post_request_async: {}", url); - - let (response, status_code) = self.post_request(url, body).await; - eprintln!("response: {}", response); - assert!(response["updateId"].as_u64().is_some()); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("put_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::put() - .uri(url) - .set_json(&body) - .to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("put_request_async: {}", url); - - let (response, status_code) = self.put_request(url, body).await; - assert!(response["updateId"].as_u64().is_some()); - assert_eq!(status_code, 202); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("delete_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::delete().uri(url).to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("delete_request_async: {}", url); - - let (response, status_code) = self.delete_request(url).await; - assert!(response["updateId"].as_u64().is_some()); - assert_eq!(status_code, 202); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - // All Routes - - pub async fn list_indexes(&mut self) -> (Value, StatusCode) { - self.get_request("/indexes").await - } - - pub async fn create_index(&mut self, body: Value) -> (Value, StatusCode) { - self.post_request("/indexes", body).await - } - - pub async fn search_multi_index(&mut self, query: &str) -> (Value, StatusCode) { - let url = format!("/indexes/search?{}", query); - self.get_request(&url).await - } - - pub async fn get_index(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.get_request(&url).await - } - - pub async fn update_index(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.put_request(&url, body).await - } - - pub async fn delete_index(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.delete_request(&url).await - } - - pub async fn search_get(&mut self, query: &str) -> (Value, StatusCode) { - let url = format!("/indexes/{}/search?{}", self.uid, query); - self.get_request(&url).await - } - - pub async fn search_post(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/search", self.uid); - self.post_request(&url, body).await - } - - pub async fn get_all_updates_status(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/updates", self.uid); - self.get_request(&url).await - } - - pub async fn get_update_status(&mut self, update_id: u64) -> (Value, StatusCode) { - let url = format!("/indexes/{}/updates/{}", self.uid, update_id); - self.get_request(&url).await - } - - pub async fn get_all_documents(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", self.uid); - self.get_request(&url).await - } - - pub async fn add_or_replace_multiple_documents(&mut self, body: Value) { - let url = format!("/indexes/{}/documents", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn add_or_replace_multiple_documents_sync( - &mut self, - body: Value, - ) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", self.uid); - self.post_request(&url, body).await - } - - pub async fn add_or_update_multiple_documents(&mut self, body: Value) { - let url = format!("/indexes/{}/documents", self.uid); - self.put_request_async(&url, body).await; - } - - pub async fn clear_all_documents(&mut self) { - let url = format!("/indexes/{}/documents", self.uid); - self.delete_request_async(&url).await; - } - - pub async fn get_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { - let url = format!( - "/indexes/{}/documents/{}", - self.uid, - document_id.to_string() - ); - self.get_request(&url).await - } - - pub async fn delete_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { - let url = format!( - "/indexes/{}/documents/{}", - self.uid, - document_id.to_string() - ); - self.delete_request_async(&url).await - } - - pub async fn delete_multiple_documents(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents/delete-batch", self.uid); - self.post_request_async(&url, body).await - } - - pub async fn get_all_settings(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.get_request(&url).await - } - - pub async fn update_all_settings(&mut self, body: Value) { - let url = format!("/indexes/{}/settings", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_all_settings_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_all_settings(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_ranking_rules(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.get_request(&url).await - } - - pub async fn update_ranking_rules(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_ranking_rules_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_ranking_rules(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_distinct_attribute(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.get_request(&url).await - } - - pub async fn update_distinct_attribute(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_distinct_attribute_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_primary_key(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/primary_key", self.uid); - self.get_request(&url).await - } - - pub async fn get_searchable_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.get_request(&url).await - } - - pub async fn update_searchable_attributes(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_searchable_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_displayed_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.get_request(&url).await - } - - pub async fn update_displayed_attributes(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_displayed_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_attributes_for_faceting(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.get_request(&url).await - } - - pub async fn update_attributes_for_faceting(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_attributes_for_faceting_sync( - &mut self, - body: Value, - ) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_attributes_for_faceting(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_synonyms(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.get_request(&url).await - } - - pub async fn update_synonyms(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_synonyms_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_synonyms(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_stop_words(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.get_request(&url).await - } - - pub async fn update_stop_words(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_stop_words_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_stop_words(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_index_stats(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/stats", self.uid); - self.get_request(&url).await - } - - pub async fn list_keys(&mut self) -> (Value, StatusCode) { - self.get_request("/keys").await - } - - pub async fn get_health(&mut self) -> (Value, StatusCode) { - self.get_request("/health").await - } - - pub async fn update_health(&mut self, body: Value) -> (Value, StatusCode) { - self.put_request("/health", body).await - } - - pub async fn get_version(&mut self) -> (Value, StatusCode) { - self.get_request("/version").await - } - - pub async fn get_sys_info(&mut self) -> (Value, StatusCode) { - self.get_request("/sys-info").await - } - - pub async fn get_sys_info_pretty(&mut self) -> (Value, StatusCode) { - self.get_request("/sys-info/pretty").await - } - - pub async fn trigger_dump(&self) -> (Value, StatusCode) { - self.post_request("/dumps", Value::Null).await - } - - pub async fn get_dump_status(&mut self, dump_uid: &str) -> (Value, StatusCode) { - let url = format!("/dumps/{}/status", dump_uid); - self.get_request(&url).await - } - - pub async fn trigger_dump_importation(&mut self, dump_uid: &str) -> (Value, StatusCode) { - let url = format!("/dumps/{}/import", dump_uid); - self.get_request(&url).await - } -} diff --git a/tests/dashboard.rs b/tests/dashboard.rs deleted file mode 100644 index 2dbaf8f7d..000000000 --- a/tests/dashboard.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn dashboard() { - let mut server = common::Server::with_uid("movies"); - - let (_response, status_code) = server.get_request("/").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("/bulma.min.css").await; - assert_eq!(status_code, 200); -} diff --git a/tests/documents_add.rs b/tests/documents_add.rs deleted file mode 100644 index 382a1ed43..000000000 --- a/tests/documents_add.rs +++ /dev/null @@ -1,222 +0,0 @@ -use serde_json::json; - -mod common; - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/519 -#[actix_rt::test] -async fn check_add_documents_with_primary_key_param() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add documents - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/568 -#[actix_rt::test] -async fn check_add_documents_with_nested_boolean() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a boolean in a nested object - - let body = json!([{ - "id": 12161, - "created_at": "2019-04-10T14:57:57.522Z", - "foo": { - "bar": { - "id": 121, - "crash": false - }, - "id": 45912 - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/571 -#[actix_rt::test] -async fn check_add_documents_with_nested_null() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a null in a nested object - - let body = json!([{ - "id": 0, - "foo": { - "bar": null - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/574 -#[actix_rt::test] -async fn check_add_documents_with_nested_sequence() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a seq in a nested object - - let body = json!([{ - "id": 0, - "foo": { - "bar": [123,456], - "fez": [{ - "id": 255, - "baz": "leesz", - "fuzz": { - "fax": [234] - }, - "sas": [] - }], - "foz": [{ - "id": 255, - "baz": "leesz", - "fuzz": { - "fax": [234] - }, - "sas": [] - }, - { - "id": 256, - "baz": "loss", - "fuzz": { - "fax": [235] - }, - "sas": [321, 321] - }] - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body.clone()).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); - - let url = "/indexes/tasks/search?q=leesz"; - let (response, status_code) = server.get_request(&url).await; - assert_eq!(status_code, 200); - assert_eq!(response["hits"], body); -} - -#[actix_rt::test] -// test sample from #807 -async fn add_document_with_long_field() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let body = json!([{ - "documentId":"de1c2adbb897effdfe0deae32a01035e46f932ce", - "rank":1, - "relurl":"/configuration/app/web.html#locations", - "section":"Web", - "site":"docs", - "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", - "title":"Locations", - "url":"/configuration/app/web.html#locations" - }]); - server.add_or_replace_multiple_documents(body).await; - let (response, _status) = server - .search_post(json!({ "q": "request_buffering" })) - .await; - assert!(!response["hits"].as_array().unwrap().is_empty()); -} - -#[actix_rt::test] -async fn documents_with_same_id_are_overwritten() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test"})).await; - let documents = json!([ - { - "id": 1, - "content": "test1" - }, - { - "id": 1, - "content": "test2" - }, - ]); - server.add_or_replace_multiple_documents(documents).await; - let (response, _status) = server.get_all_documents().await; - assert_eq!(response.as_array().unwrap().len(), 1); - assert_eq!( - response.as_array().unwrap()[0].as_object().unwrap()["content"], - "test2" - ); -} diff --git a/tests/documents_delete.rs b/tests/documents_delete.rs deleted file mode 100644 index 4353a5355..000000000 --- a/tests/documents_delete.rs +++ /dev/null @@ -1,67 +0,0 @@ -mod common; - -use serde_json::json; - -#[actix_rt::test] -async fn delete() { - let mut server = common::Server::test_server().await; - - let (_response, status_code) = server.get_document(50).await; - assert_eq!(status_code, 200); - - server.delete_document(50).await; - - let (_response, status_code) = server.get_document(50).await; - assert_eq!(status_code, 404); -} - -// Resolve the issue https://github.com/meilisearch/MeiliSearch/issues/493 -#[actix_rt::test] -async fn delete_batch() { - let mut server = common::Server::test_server().await; - - let doc_ids = vec!(50, 55, 60); - for doc_id in &doc_ids { - let (_response, status_code) = server.get_document(doc_id).await; - assert_eq!(status_code, 200); - } - - let body = serde_json::json!(&doc_ids); - server.delete_multiple_documents(body).await; - - for doc_id in &doc_ids { - let (_response, status_code) = server.get_document(doc_id).await; - assert_eq!(status_code, 404); - } -} - -#[actix_rt::test] -async fn text_clear_all_placeholder_search() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - - server.create_index(body).await; - let settings = json!({ - "attributesForFaceting": ["genre"], - }); - - server.update_all_settings(settings).await; - - let documents = json!([ - { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance" }, - { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" }, - { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy" }, - { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, - { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, - { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams" } - ]); - - server.add_or_update_multiple_documents(documents).await; - server.clear_all_documents().await; - let (response, _) = server.search_post(json!({ "q": "", "facetsDistribution": ["genre"] })).await; - assert_eq!(response["nbHits"], 0); - let (response, _) = server.search_post(json!({ "q": "" })).await; - assert_eq!(response["nbHits"], 0); -} diff --git a/tests/documents_get.rs b/tests/documents_get.rs deleted file mode 100644 index 35e04f494..000000000 --- a/tests/documents_get.rs +++ /dev/null @@ -1,23 +0,0 @@ -use serde_json::json; -use actix_web::http::StatusCode; - -mod common; - -#[actix_rt::test] -async fn get_documents_from_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.get_all_documents().await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); - assert_eq!(response["errorLink"], "https://docs.meilisearch.com/errors#index_not_found"); -} - -#[actix_rt::test] -async fn get_empty_documents_list() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let (response, status) = server.get_all_documents().await; - assert_eq!(status, StatusCode::OK); - assert!(response.as_array().unwrap().is_empty()); -} diff --git a/tests/dump.rs b/tests/dump.rs deleted file mode 100644 index 701b754aa..000000000 --- a/tests/dump.rs +++ /dev/null @@ -1,395 +0,0 @@ -use assert_json_diff::{assert_json_eq, assert_json_include}; -use meilisearch_http::helpers::compression; -use serde_json::{json, Value}; -use std::fs::File; -use std::path::Path; -use std::thread; -use std::time::Duration; -use tempfile::TempDir; - -#[macro_use] mod common; - -async fn trigger_and_wait_dump(server: &mut common::Server) -> String { - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - for _ in 0..20 as u8 { - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - assert_ne!(value["status"].as_str(), Some("dump_process_failed")); - - if value["status"].as_str() == Some("done") { return dump_uid } - thread::sleep(Duration::from_millis(100)); - } - - unreachable!("dump creation runned out of time") -} - -fn current_db_version() -> (String, String, String) { - let current_version_major = env!("CARGO_PKG_VERSION_MAJOR").to_string(); - let current_version_minor = env!("CARGO_PKG_VERSION_MINOR").to_string(); - let current_version_patch = env!("CARGO_PKG_VERSION_PATCH").to_string(); - - (current_version_major, current_version_minor, current_version_patch) -} - -fn current_dump_version() -> String { - "V1".into() -} - -fn read_all_jsonline(r: R) -> Value { - let deserializer = serde_json::Deserializer::from_reader(r); - let iterator = deserializer.into_iter::(); - - json!(iterator.map(|v| v.unwrap()).collect::>()) -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_should_return_ok() { - let server = common::Server::test_server().await; - - let (_, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_twice_should_return_conflict() { - let server = common::Server::test_server().await; - - let expected = json!({ - "message": "Another dump is already in progress", - "errorCode": "dump_already_in_progress", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" - }); - - let (_, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let (value, status_code) = server.trigger_dump().await; - - - assert_json_eq!(expected, value, ordered: false); - assert_eq!(status_code, 409); -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_concurently_should_return_conflict() { - let server = common::Server::test_server().await; - - let expected = json!({ - "message": "Another dump is already in progress", - "errorCode": "dump_already_in_progress", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" - }); - - let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump()); - - assert_json_eq!(expected, value_2, ordered: false); - assert_eq!(status_code_2, 409); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_early_should_return_in_progress() { - let mut server = common::Server::test_server().await; - - - - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - let expected = json!({ - "uid": dump_uid, - "status": "in_progress" - }); - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_should_return_done() { - let mut server = common::Server::test_server().await; - - - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let expected = json!({ - "uid": dump_uid.clone(), - "status": "done" - }); - - thread::sleep(Duration::from_secs(1)); // wait dump until process end - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_should_return_error_provoking_it() { - let mut server = common::Server::test_server().await; - - - let (value, status_code) = server.trigger_dump().await; - - // removing destination directory provoking `No such file or directory` error - std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap(); - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let expected = json!({ - "uid": dump_uid.clone(), - "status": "failed", - "message": "Dump process failed: compressing dump; No such file or directory (os error 2)", - "errorCode": "dump_process_failed", - "errorType": "internal_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed" - }); - - thread::sleep(Duration::from_secs(1)); // wait dump until process end - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_metadata_should_be_valid() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "uid": "test2", - "primaryKey": "test2_id", - }); - - server.create_index(body).await; - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); - let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); - - // fields are randomly ordered - metadata.get_mut("indexes").unwrap() - .as_array_mut().unwrap() - .sort_by(|a, b| - a.get("uid").unwrap().as_str().cmp(&b.get("uid").unwrap().as_str()) - ); - - let (major, minor, patch) = current_db_version(); - - let expected = json!({ - "indexes": [{ - "uid": "test", - "primaryKey": "id", - }, { - "uid": "test2", - "primaryKey": "test2_id", - } - ], - "dbVersion": format!("{}.{}.{}", major, minor, patch), - "dumpVersion": current_dump_version() - }); - - assert_json_include!(expected: expected, actual: metadata); -} - -#[actix_rt::test] -#[ignore] -async fn dump_gzip_should_have_been_created() { - let mut server = common::Server::test_server().await; - - - let dump_uid = trigger_and_wait_dump(&mut server).await; - let dumps_dir = Path::new(&server.data().dumps_dir); - - let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid)); - assert!(File::open(compressed_path).is_ok()); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_settings_should_be_valid() { - let mut server = common::Server::test_server().await; - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": "email", - "searchableAttributes": [ - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "displayedAttributes": [ - "id", - "isActive", - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "stopWords": [ - "in", - "ad" - ], - "synonyms": { - "wolverine": ["xmen", "logan"], - "logan": ["wolverine", "xmen"] - }, - "attributesForFaceting": [ - "gender", - "color", - "tags" - ] - }); - - server.update_all_settings(expected.clone()).await; - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); - let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); - - assert_json_eq!(expected, settings, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_documents_should_be_valid() { - let mut server = common::Server::test_server().await; - - let dataset = include_bytes!("assets/dumps/v1/test/documents.jsonl"); - let mut slice: &[u8] = dataset; - - let expected: Value = read_all_jsonline(&mut slice); - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); - let documents = read_all_jsonline(file); - - assert_json_eq!(expected, documents, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_updates_should_be_valid() { - let mut server = common::Server::test_server().await; - - let dataset = include_bytes!("assets/dumps/v1/test/updates.jsonl"); - let mut slice: &[u8] = dataset; - - let expected: Value = read_all_jsonline(&mut slice); - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); - let mut updates = read_all_jsonline(file); - - - // hotfix until #943 is fixed (https://github.com/meilisearch/MeiliSearch/issues/943) - updates.as_array_mut().unwrap() - .get_mut(0).unwrap() - .get_mut("type").unwrap() - .get_mut("settings").unwrap() - .get_mut("displayed_attributes").unwrap() - .get_mut("Update").unwrap() - .as_array_mut().unwrap().sort_by(|a, b| a.as_str().cmp(&b.as_str())); - - eprintln!("{}\n", updates.to_string()); - eprintln!("{}", expected.to_string()); - assert_json_include!(expected: expected, actual: updates); -} - -#[actix_rt::test] -#[ignore] -async fn get_unexisting_dump_status_should_return_not_found() { - let mut server = common::Server::test_server().await; - - let (_, status_code) = server.get_dump_status("4242").await; - - assert_eq!(status_code, 404); -} diff --git a/tests/errors.rs b/tests/errors.rs deleted file mode 100644 index e11483356..000000000 --- a/tests/errors.rs +++ /dev/null @@ -1,200 +0,0 @@ -mod common; - -use std::thread; -use std::time::Duration; - -use actix_http::http::StatusCode; -use serde_json::{json, Map, Value}; - -macro_rules! assert_error { - ($code:literal, $type:literal, $status:path, $req:expr) => { - let (response, status_code) = $req; - assert_eq!(status_code, $status); - assert_eq!(response["errorCode"].as_str().unwrap(), $code); - assert_eq!(response["errorType"].as_str().unwrap(), $type); - }; -} - -macro_rules! assert_error_async { - ($code:literal, $type:literal, $server:expr, $req:expr) => { - let (response, _) = $req; - let update_id = response["updateId"].as_u64().unwrap(); - for _ in 1..10 { - let (response, status_code) = $server.get_update_status(update_id).await; - assert_eq!(status_code, StatusCode::OK); - if response["status"] == "processed" || response["status"] == "failed" { - println!("response: {}", response); - assert_eq!(response["status"], "failed"); - assert_eq!(response["errorCode"], $code); - assert_eq!(response["errorType"], $type); - return - } - thread::sleep(Duration::from_secs(1)); - } - }; -} - -#[actix_rt::test] -async fn index_already_exists_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test" - }); - let (response, status_code) = server.create_index(body.clone()).await; - println!("{}", response); - assert_eq!(status_code, StatusCode::CREATED); - - let (response, status_code) = server.create_index(body.clone()).await; - println!("{}", response); - - assert_error!( - "index_already_exists", - "invalid_request_error", - StatusCode::BAD_REQUEST, - (response, status_code)); -} - -#[actix_rt::test] -async fn index_not_found_error() { - let mut server = common::Server::with_uid("test"); - assert_error!( - "index_not_found", - "invalid_request_error", - StatusCode::NOT_FOUND, - server.get_index().await); -} - -#[actix_rt::test] -async fn primary_key_already_present_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "test" - }); - server.create_index(body.clone()).await; - let body = json!({ - "primaryKey": "t" - }); - assert_error!( - "primary_key_already_present", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.update_index(body).await); -} - -#[actix_rt::test] -async fn max_field_limit_exceeded_error() { - let mut server = common::Server::test_server().await; - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - let mut doc = Map::with_capacity(70_000); - doc.insert("id".into(), Value::String("foo".into())); - for i in 0..69_999 { - doc.insert(format!("field{}", i), Value::String("foo".into())); - } - let docs = json!([doc]); - assert_error_async!( - "max_fields_limit_exceeded", - "invalid_request_error", - server, - server.add_or_replace_multiple_documents_sync(docs).await); -} - -#[actix_rt::test] -async fn missing_document_id() { - let mut server = common::Server::test_server().await; - let body = json!({ - "uid": "test", - "primaryKey": "test" - }); - server.create_index(body).await; - let docs = json!([ - { - "foo": "bar", - } - ]); - assert_error_async!( - "missing_document_id", - "invalid_request_error", - server, - server.add_or_replace_multiple_documents_sync(docs).await); -} - -#[actix_rt::test] -async fn facet_error() { - let mut server = common::Server::test_server().await; - let search = json!({ - "q": "foo", - "facetFilters": ["test:hello"] - }); - assert_error!( - "invalid_facet", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(search).await); -} - -#[actix_rt::test] -async fn filters_error() { - let mut server = common::Server::test_server().await; - let search = json!({ - "q": "foo", - "filters": "fo:12" - }); - assert_error!( - "invalid_filter", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(search).await); -} - -#[actix_rt::test] -async fn bad_request_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "foo": "bar", - }); - assert_error!( - "bad_request", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(body).await); -} - -#[actix_rt::test] -async fn document_not_found_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({"uid": "test"})).await; - assert_error!( - "document_not_found", - "invalid_request_error", - StatusCode::NOT_FOUND, - server.get_document(100).await); -} - -#[actix_rt::test] -async fn payload_too_large_error() { - let mut server = common::Server::with_uid("test"); - let bigvec = vec![0u64; 10_000_000]; // 80mb - assert_error!( - "payload_too_large", - "invalid_request_error", - StatusCode::PAYLOAD_TOO_LARGE, - server.create_index(json!(bigvec)).await); -} - -#[actix_rt::test] -async fn missing_primary_key_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({"uid": "test"})).await; - let document = json!([{ - "content": "test" - }]); - assert_error!( - "missing_primary_key", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.add_or_replace_multiple_documents_sync(document).await); -} diff --git a/tests/health.rs b/tests/health.rs deleted file mode 100644 index f72127431..000000000 --- a/tests/health.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn test_healthyness() { - let mut server = common::Server::with_uid("movies"); - - // Check that the server is healthy - - let (_response, status_code) = server.get_health().await; - assert_eq!(status_code, 204); -} diff --git a/tests/index.rs b/tests/index.rs deleted file mode 100644 index 271507e03..000000000 --- a/tests/index.rs +++ /dev/null @@ -1,809 +0,0 @@ -use actix_web::http::StatusCode; -use assert_json_diff::assert_json_eq; -use serde_json::{json, Value}; - -mod common; - -#[actix_rt::test] -async fn create_index_with_name() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn create_index_with_uid() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body.clone()).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid, "movies"); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 1.5 verify that error is thrown when trying to create the same index - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - assert_eq!( - response["errorCode"].as_str().unwrap(), - "index_already_exists" - ); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn create_index_with_name_and_uid() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "Films", - "uid": "fr_movies", - }); - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "Films"); - assert_eq!(r1_uid, "fr_movies"); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn rename_index() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 6); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Update an index name - - let body = json!({ - "name": "TV Shows", - }); - - let (res2_value, status_code) = server.update_index(body).await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_object().unwrap().len(), 5); - let r2_name = res2_value["name"].as_str().unwrap(); - let r2_uid = res2_value["uid"].as_str().unwrap(); - let r2_created_at = res2_value["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, "TV Shows"); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at, r1_created_at); - assert!(r2_updated_at.len() > 1); - - // 3 - Check the list of indexes - - let (res3_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res3_value.as_array().unwrap().len(), 1); - assert_eq!(res3_value[0].as_object().unwrap().len(), 5); - let r3_name = res3_value[0]["name"].as_str().unwrap(); - let r3_uid = res3_value[0]["uid"].as_str().unwrap(); - let r3_created_at = res3_value[0]["createdAt"].as_str().unwrap(); - let r3_updated_at = res3_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r3_name, r2_name); - assert_eq!(r3_uid.len(), r1_uid.len()); - assert_eq!(r3_created_at.len(), r1_created_at.len()); - assert_eq!(r3_updated_at.len(), r2_updated_at.len()); -} - -#[actix_rt::test] -async fn delete_index_and_recreate_it() { - let mut server = common::Server::with_uid("movies"); - - // 0 - delete unexisting index is error - - let (response, status_code) = server.delete_request("/indexes/test").await; - assert_eq!(status_code, 404); - assert_eq!(&response["errorCode"], "index_not_found"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 6); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); - - // 3- Delete an index - - let (_res2_value, status_code) = server.delete_index().await; - - assert_eq!(status_code, 204); - - // 4 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 0); - - // 5 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 6 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn check_multiples_indexes() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_0_name = res2_value[0]["name"].as_str().unwrap(); - let r2_0_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_0_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_0_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_0_name, r1_name); - assert_eq!(r2_0_uid.len(), r1_uid.len()); - assert_eq!(r2_0_created_at.len(), r1_created_at.len()); - assert_eq!(r2_0_updated_at.len(), r1_updated_at.len()); - - // 3 - Create a new index - - let body = json!({ - "name": "films", - }); - - let (res3_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res3_value.as_object().unwrap().len(), 5); - let r3_name = res3_value["name"].as_str().unwrap(); - let r3_uid = res3_value["uid"].as_str().unwrap(); - let r3_created_at = res3_value["createdAt"].as_str().unwrap(); - let r3_updated_at = res3_value["updatedAt"].as_str().unwrap(); - assert_eq!(r3_name, "films"); - assert_eq!(r3_uid.len(), 8); - assert!(r3_created_at.len() > 1); - assert!(r3_updated_at.len() > 1); - - // 4 - Check the list of indexes - - let (res4_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res4_value.as_array().unwrap().len(), 2); - assert_eq!(res4_value[0].as_object().unwrap().len(), 5); - let r4_0_name = res4_value[0]["name"].as_str().unwrap(); - let r4_0_uid = res4_value[0]["uid"].as_str().unwrap(); - let r4_0_created_at = res4_value[0]["createdAt"].as_str().unwrap(); - let r4_0_updated_at = res4_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(res4_value[1].as_object().unwrap().len(), 5); - let r4_1_name = res4_value[1]["name"].as_str().unwrap(); - let r4_1_uid = res4_value[1]["uid"].as_str().unwrap(); - let r4_1_created_at = res4_value[1]["createdAt"].as_str().unwrap(); - let r4_1_updated_at = res4_value[1]["updatedAt"].as_str().unwrap(); - if r4_0_name == r1_name { - assert_eq!(r4_0_name, r1_name); - assert_eq!(r4_0_uid.len(), r1_uid.len()); - assert_eq!(r4_0_created_at.len(), r1_created_at.len()); - assert_eq!(r4_0_updated_at.len(), r1_updated_at.len()); - } else { - assert_eq!(r4_0_name, r3_name); - assert_eq!(r4_0_uid.len(), r3_uid.len()); - assert_eq!(r4_0_created_at.len(), r3_created_at.len()); - assert_eq!(r4_0_updated_at.len(), r3_updated_at.len()); - } - if r4_1_name == r1_name { - assert_eq!(r4_1_name, r1_name); - assert_eq!(r4_1_uid.len(), r1_uid.len()); - assert_eq!(r4_1_created_at.len(), r1_created_at.len()); - assert_eq!(r4_1_updated_at.len(), r1_updated_at.len()); - } else { - assert_eq!(r4_1_name, r3_name); - assert_eq!(r4_1_uid.len(), r3_uid.len()); - assert_eq!(r4_1_created_at.len(), r3_created_at.len()); - assert_eq!(r4_1_updated_at.len(), r3_updated_at.len()); - } -} - -#[actix_rt::test] -async fn create_index_failed() { - let mut server = common::Server::with_uid("movies"); - - // 2 - Push index creation with empty json body - - let body = json!({}); - - let (res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = res_value["message"].as_str().unwrap(); - assert_eq!(res_value.as_object().unwrap().len(), 4); - assert_eq!(message, "Index creation must have an uid"); - - // 3 - Create a index with extra data - - let body = json!({ - "name": "movies", - "active": true - }); - - let (_res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - - // 3 - Create a index with wrong data type - - let body = json!({ - "name": "movies", - "uid": 0 - }); - - let (_res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); -} - -// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/492 -#[actix_rt::test] -async fn create_index_with_primary_key_and_index() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index - - let body = json!({ - "uid": "movies", - "primaryKey": "id", - }); - - let (_response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - - // 2 - Add content - - let body = json!([{ - "id": 123, - "text": "The mask" - }]); - - server.add_or_replace_multiple_documents(body.clone()).await; - - // 3 - Retreive document - - let (response, _status_code) = server.get_document(123).await; - - let expect = json!({ - "id": 123, - "text": "The mask" - }); - - assert_json_eq!(response, expect, ordered: false); -} - -// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/497 -// Test when the given index uid is not valid -// Should have a 400 status code -// Should have the right error message -#[actix_rt::test] -async fn create_index_with_invalid_uid() { - let mut server = common::Server::with_uid(""); - - // 1 - Create the index with invalid uid - - let body = json!({ - "uid": "the movies" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 2 - Create the index with invalid uid - - let body = json!({ - "uid": "%$#" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 3 - Create the index with invalid uid - - let body = json!({ - "uid": "the~movies" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 4 - Create the index with invalid uid - - let body = json!({ - "uid": "🎉" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); -} - -// Test that it's possible to add primary_key if it's not already set on index creation -#[actix_rt::test] -async fn create_index_and_add_indentifier_after() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Update the index and add an primary_key. - - let body = json!({ - "primaryKey": "id", - }); - - let (response, status_code) = server.update_index(body).await; - assert_eq!(status_code, 200); - eprintln!("response: {:#?}", response); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); - - // 3 - Get index to verify if the primary_key is good - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test that it's impossible to change the primary_key -#[actix_rt::test] -async fn create_index_and_update_indentifier_after() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - "primaryKey": "id", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); - - // 2 - Update the index and add an primary_key. - - let body = json!({ - "primaryKey": "skuid", - }); - - let (_response, status_code) = server.update_index(body).await; - assert_eq!(status_code, 400); - - // 3 - Get index to verify if the primary_key still the first one - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test that schema inference work well -#[actix_rt::test] -async fn create_index_without_primary_key_and_add_document() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document - - let body = json!([{ - "id": 123, - "title": "I'm a legend", - }]); - - server.add_or_update_multiple_documents(body).await; - - // 3 - Get index to verify if the primary_key is good - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test search with no primary_key -#[actix_rt::test] -async fn create_index_without_primary_key_and_search() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Search - - let query = "q=captain&limit=3"; - - let (response, status_code) = server.search_get(&query).await; - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 0); -} - -// Test the error message when we push an document update and impossibility to find primary key -// Test issue https://github.com/meilisearch/MeiliSearch/issues/517 -#[actix_rt::test] -async fn check_add_documents_without_primary_key() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2- Add document - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let (response, status_code) = server.add_or_replace_multiple_documents_sync(body).await; - - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(response["errorCode"], "missing_primary_key"); - assert_eq!(status_code, 400); -} - -#[actix_rt::test] -async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { - let mut server = common::Server::with_uid("movies"); - - let body = json!({ - "uid": "movies", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let dataset = include_bytes!("./assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - // 2. Index the documents from movies.json, present inside of assets directory - server.add_or_replace_multiple_documents(body).await; - - // 3. Fetch the status of the indexing done above. - let (response, status_code) = server.get_all_updates_status().await; - - // 4. Verify the fetch is successful and indexing status is 'processed' - assert_eq!(status_code, 200); - assert_eq!(response[0]["status"], "processed"); -} - -#[actix_rt::test] -async fn get_empty_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server.list_indexes().await; - assert!(response.as_array().unwrap().is_empty()); -} - -#[actix_rt::test] -async fn create_and_list_multiple_indices() { - let mut server = common::Server::with_uid("test"); - for i in 0..10 { - server - .create_index(json!({ "uid": format!("test{}", i) })) - .await; - } - let (response, _status) = server.list_indexes().await; - assert_eq!(response.as_array().unwrap().len(), 10); -} - -#[actix_rt::test] -async fn get_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.get_index().await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn create_index_twice_is_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let (response, status) = server.create_index(json!({ "uid": "test" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "index_already_exists"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn badly_formatted_index_name_is_error() { - let mut server = common::Server::with_uid("$__test"); - let (response, status) = server.create_index(json!({ "uid": "$__test" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "invalid_index_uid"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn correct_response_no_primary_key_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server.create_index(json!({ "uid": "test" })).await; - assert_eq!(response["primaryKey"], Value::Null); -} - -#[actix_rt::test] -async fn correct_response_with_primary_key_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server - .create_index(json!({ "uid": "test", "primaryKey": "test" })) - .await; - assert_eq!(response["primaryKey"], "test"); -} - -#[actix_rt::test] -async fn udpate_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.update_index(json!({ "primaryKey": "foobar" })).await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn update_existing_primary_key_is_error() { - let mut server = common::Server::with_uid("test"); - server - .create_index(json!({ "uid": "test", "primaryKey": "key" })) - .await; - let (response, status) = server.update_index(json!({ "primaryKey": "test2" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "primary_key_already_present"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn test_facets_distribution_attribute() { - let mut server = common::Server::test_server().await; - - let (response, _status_code) = server.get_index_stats().await; - - let expected = json!({ - "isIndexing": false, - "numberOfDocuments":77, - "fieldsDistribution":{ - "age":77, - "gender":77, - "phone":77, - "name":77, - "registered":77, - "latitude":77, - "email":77, - "tags":77, - "longitude":77, - "color":77, - "address":77, - "balance":77, - "about":77, - "picture":77, - }, - }); - - assert_json_eq!(expected, response, ordered: true); -} diff --git a/tests/index_update.rs b/tests/index_update.rs deleted file mode 100644 index df4639252..000000000 --- a/tests/index_update.rs +++ /dev/null @@ -1,200 +0,0 @@ -use serde_json::json; -use serde_json::Value; -use assert_json_diff::assert_json_include; - -mod common; - -#[actix_rt::test] -async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let dataset = include_bytes!("assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - // 2. Index the documents from movies.json, present inside of assets directory - server.add_or_replace_multiple_documents(body).await; - - // 3. Fetch the status of the indexing done above. - let (response, status_code) = server.get_all_updates_status().await; - - // 4. Verify the fetch is successful and indexing status is 'processed' - assert_eq!(status_code, 200); - assert_eq!(response[0]["status"], "processed"); -} - -#[actix_rt::test] -async fn return_error_when_get_update_status_of_unexisting_index() { - let mut server = common::Server::with_uid("test"); - - // 1. Fetch the status of unexisting index. - let (_, status_code) = server.get_all_updates_status().await; - - // 2. Verify the fetch returned 404 - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn return_empty_when_get_update_status_of_empty_index() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2. Fetch the status of empty index. - let (response, status_code) = server.get_all_updates_status().await; - - // 3. Verify the fetch is successful, and no document are returned - assert_eq!(status_code, 200); - assert_eq!(response, json!([])); -} - -#[actix_rt::test] -async fn return_update_status_of_pushed_documents() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - - let bodies = vec![ - json!([{ - "title": "Test", - "comment": "comment test" - }]), - json!([{ - "title": "Test1", - "comment": "comment test1" - }]), - json!([{ - "title": "Test2", - "comment": "comment test2" - }]), - ]; - - let mut update_ids = Vec::new(); - - let url = "/indexes/test/documents?primaryKey=title"; - for body in bodies { - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - update_ids.push(update_id); - } - - // 2. Fetch the status of index. - let (response, status_code) = server.get_all_updates_status().await; - - // 3. Verify the fetch is successful, and updates are returned - - let expected = json!([{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[0] - },{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[1] - },{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[2] - },]); - - assert_eq!(status_code, 200); - assert_json_include!(actual: json!(response), expected: expected); -} - -#[actix_rt::test] -async fn return_error_if_index_does_not_exist() { - let mut server = common::Server::with_uid("test"); - - let (response, status_code) = server.get_update_status(42).await; - - assert_eq!(status_code, 404); - assert_eq!(response["errorCode"], "index_not_found"); -} - -#[actix_rt::test] -async fn return_error_if_update_does_not_exist() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let (response, status_code) = server.get_update_status(42).await; - - assert_eq!(status_code, 404); - assert_eq!(response["errorCode"], "not_found"); -} - -#[actix_rt::test] -async fn should_return_existing_update() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/test/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - - let update_id = response["updateId"].as_u64().unwrap(); - - let expected = json!({ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_id - }); - - let (response, status_code) = server.get_update_status(update_id).await; - - assert_eq!(status_code, 200); - assert_json_include!(actual: json!(response), expected: expected); -} diff --git a/tests/lazy_index_creation.rs b/tests/lazy_index_creation.rs deleted file mode 100644 index 6730db82e..000000000 --- a/tests/lazy_index_creation.rs +++ /dev/null @@ -1,446 +0,0 @@ -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Add documents - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents_and_discover_pk() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Add documents - - let body = json!([{ - "id": 1, - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents_with_wrong_name() { - let server = common::Server::with_uid("wrong&name"); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/wrong&name/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_index_uid"); -} - -#[actix_rt::test] -async fn create_index_lazy_add_documents_failed() { - let mut server = common::Server::with_uid("wrong&name"); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/wrong&name/documents"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_index_uid"); - - let (_, status_code) = server.get_index().await; - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_settings() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_settings_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "other", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "anotherSettings": ["name"], - }); - - let (_, status_code) = server.update_all_settings_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_ranking_rules() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_ranking_rules_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": 123, - }); - - let (_, status_code) = server.update_ranking_rules_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_distinct_attribute() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!("type"); - - server.update_distinct_attribute(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_distinct_attribute_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (resp, status_code) = server.update_distinct_attribute_sync(body.clone()).await; - eprintln!("resp: {:?}", resp); - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (resp, status_code) = server.get_all_settings().await; - eprintln!("resp: {:?}", resp); - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_searchable_attributes() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_searchable_attributes(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_searchable_attributes_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_searchable_attributes_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_displayed_attributes() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_displayed_attributes(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_displayed_attributes_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_displayed_attributes_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_attributes_for_faceting() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_attributes_for_faceting(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_attributes_for_faceting_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server - .update_attributes_for_faceting_sync(body.clone()) - .await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_synonyms() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "road": ["street", "avenue"], - "street": ["avenue"], - }); - - server.update_synonyms(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_synonyms_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_synonyms_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_stop_words() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["le", "la", "les"]); - - server.update_stop_words(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_stop_words_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_stop_words_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} diff --git a/tests/placeholder_search.rs b/tests/placeholder_search.rs deleted file mode 100644 index 048ab7f8b..000000000 --- a/tests/placeholder_search.rs +++ /dev/null @@ -1,629 +0,0 @@ -use std::convert::Into; - -use serde_json::json; -use serde_json::Value; -use std::cell::RefCell; -use std::sync::Mutex; - -#[macro_use] -mod common; - -#[actix_rt::test] -async fn placeholder_search_with_limit() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "limit": 3 - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 3); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_offset() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 6, - }); - - // hack to take a value out of macro (must implement UnwindSafe) - let expected = Mutex::new(RefCell::new(Vec::new())); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - // take results at offset 3 as reference - let lock = expected.lock().unwrap(); - lock.replace(response["hits"].as_array().unwrap()[3..6].to_vec()); - }); - let expected = expected.into_inner().unwrap().into_inner(); - - let query = json!({ - "limit": 3, - "offset": 3, - }); - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let response = response["hits"].as_array().unwrap(); - assert_eq!(&expected, response); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_attribute_to_highlight_wildcard() { - // there should be no highlight in placeholder search - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 1, - "attributesToHighlight": ["*"] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let result = response["hits"].as_array().unwrap()[0].as_object().unwrap(); - for value in result.values() { - assert!(value.to_string().find("").is_none()); - } - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_matches() { - // matches is always empty - let mut server = common::Server::test_server().await; - - let query = json!({ - "matches": true - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let result = response["hits"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_object().unwrap()["_matchesInfo"].clone()) - .all(|m| m.as_object().unwrap().is_empty()); - assert!(result); - }); -} - -#[actix_rt::test] -async fn placeholder_search_witch_crop() { - // placeholder search crop always crop from beggining - let mut server = common::Server::test_server().await; - - let query = json!({ - "attributesToCrop": ["about"], - "cropLength": 20 - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - - let hits = response["hits"].as_array().unwrap(); - - for hit in hits { - let hit = hit.as_object().unwrap(); - let formatted = hit["_formatted"].as_object().unwrap(); - - let about = hit["about"].as_str().unwrap(); - let about_formatted = formatted["about"].as_str().unwrap(); - // the formatted about length should be about 20 characters long - assert!(about_formatted.len() < 20 + 10); - // the formatted part should be located at the beginning of the original one - assert_eq!(about.find(&about_formatted).unwrap(), 0); - } - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_attributes_to_retrieve() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 1, - "attributesToRetrieve": ["gender", "about"], - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hit = response["hits"].as_array().unwrap()[0].as_object().unwrap(); - assert_eq!(hit.values().count(), 2); - let _ = hit["gender"]; - let _ = hit["about"]; - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_filter() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "filters": "color='green'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green")); - }); - - let query = json!({ - "filters": "tags=bug" - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let value = Value::String(String::from("bug")); - assert!(hits - .iter() - .all(|v| v["tags"].as_array().unwrap().contains(&value))); - }); - - let query = json!({ - "filters": "color='green' AND (tags='bug' OR tags='wontfix')" - }); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let bug = Value::String(String::from("bug")); - let wontfix = Value::String(String::from("wontfix")); - assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green" - && v["tags"].as_array().unwrap().contains(&bug) - || v["tags"].as_array().unwrap().contains(&wontfix))); - }); -} - -#[actix_rt::test] -async fn placeholder_test_faceted_search_valid() { - let mut server = common::Server::test_server().await; - - // simple tests on attributes with string value - let body = json!({ - "attributesForFaceting": ["color"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "facetFilters": ["color:green"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "Green")); - }); - - let query = json!({ - "facetFilters": [["color:blue"]] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - let query = json!({ - "facetFilters": ["color:Blue"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - // test on arrays: ["tags:bug"] - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "facetFilters": ["tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())))); - }); - - // test and: ["color:blue", "tags:bug"] - let query = json!({ - "facetFilters": ["color:blue", "tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue" - && value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())))); - }); - - // test or: [["color:blue", "color:green"]] - let query = json!({ - "facetFilters": [["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue" - || value.get("color").unwrap() == "Green")); - }); - // test and-or: ["tags:bug", ["color:blue", "color:green"]] - let query = json!({ - "facetFilters": ["tags:bug", ["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())) - && (value.get("color").unwrap() == "blue" - || value.get("color").unwrap() == "Green"))); - }); -} - -#[actix_rt::test] -async fn placeholder_test_faceted_search_invalid() { - let mut server = common::Server::test_server().await; - - //no faceted attributes set - let query = json!({ - "facetFilters": ["color:blue"] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // empty arrays are error - // [] - let query = json!({ - "facetFilters": [] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // [[]] - let query = json!({ - "facetFilters": [[]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // ["color:green", []] - let query = json!({ - "facetFilters": ["color:green", []] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - - // too much depth - // [[[]]] - let query = json!({ - "facetFilters": [[[]]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // [["color:green", ["color:blue"]]] - let query = json!({ - "facetFilters": [["color:green", ["color:blue"]]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // "color:green" - let query = json!({ - "facetFilters": "color:green" - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); -} - -#[actix_rt::test] -async fn placeholder_test_facet_count() { - let mut server = common::Server::test_server().await; - - // test without facet distribution - let query = json!({}); - test_post_get_search!(server, query, |response, _status_code| { - assert!(response.get("exhaustiveFacetsCount").is_none()); - assert!(response.get("facetsDistribution").is_none()); - }); - - // test no facets set, search on color - let query = json!({ - "facetsDistribution": ["color"] - }); - test_post_get_search!(server, query.clone(), |_response, status_code| { - assert_eq!(status_code, 400); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // same as before, but now facets are set: - test_post_get_search!(server, query, |response, _status_code| { - println!("{}", response); - assert!(response.get("exhaustiveFacetsCount").is_some()); - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 1 - ); - }); - // searching on color and tags - let query = json!({ - "facetsDistribution": ["color", "tags"] - }); - test_post_get_search!(server, query, |response, _status_code| { - let facets = response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap(); - assert_eq!(facets.values().count(), 2); - assert_ne!( - !facets - .get("color") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - assert_ne!( - !facets - .get("tags") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - }); - // wildcard - let query = json!({ - "facetsDistribution": ["*"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 2 - ); - }); - // wildcard with other attributes: - let query = json!({ - "facetsDistribution": ["color", "*"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 2 - ); - }); - - // empty facet list - let query = json!({ - "facetsDistribution": [] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - }); - - // attr not set as facet passed: - let query = json!({ - "facetsDistribution": ["gender"] - }); - test_post_get_search!(server, query, |_response, status_code| { - assert_eq!(status_code, 400); - }); -} - -#[actix_rt::test] -#[should_panic] -async fn placeholder_test_bad_facet_distribution() { - let mut server = common::Server::test_server().await; - // string instead of array: - let query = json!({ - "facetsDistribution": "color" - }); - test_post_get_search!(server, query, |_response, _status_code| {}); - - // invalid value in array: - let query = json!({ - "facetsDistribution": ["color", true] - }); - test_post_get_search!(server, query, |_response, _status_code| {}); -} - -#[actix_rt::test] -async fn placeholder_test_sort() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "rankingRules": ["asc(age)"], - "attributesForFaceting": ["color"] - }); - server.update_all_settings(body).await; - let query = json!({}); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - hits.iter() - .map(|v| v["age"].as_u64().unwrap()) - .fold(0, |prev, cur| { - assert!(cur >= prev); - cur - }); - }); - - let query = json!({ - "facetFilters": ["color:green"] - }); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - hits.iter() - .map(|v| v["age"].as_u64().unwrap()) - .fold(0, |prev, cur| { - assert!(cur >= prev); - cur - }); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_empty_query() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "", - "limit": 3 - }); - - test_post_get_search!(server, query, |response, status_code| { - eprintln!("{}", response); - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 3); - }); -} - -#[actix_rt::test] -async fn test_filter_nb_hits_search_placeholder() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let documents = json!([ - { - "id": 1, - "content": "a", - "color": "green", - "size": 1, - }, - { - "id": 2, - "content": "a", - "color": "green", - "size": 2, - }, - { - "id": 3, - "content": "a", - "color": "blue", - "size": 3, - }, - ]); - - server.add_or_update_multiple_documents(documents).await; - let (response, _) = server.search_post(json!({})).await; - assert_eq!(response["nbHits"], 3); - - server.update_distinct_attribute(json!("color")).await; - - let (response, _) = server.search_post(json!({})).await; - assert_eq!(response["nbHits"], 2); - - let (response, _) = server.search_post(json!({"filters": "size < 3"})).await; - println!("result: {}", response); - assert_eq!(response["nbHits"], 1); -} diff --git a/tests/search.rs b/tests/search.rs deleted file mode 100644 index 267c98265..000000000 --- a/tests/search.rs +++ /dev/null @@ -1,1879 +0,0 @@ -use std::convert::Into; - -use assert_json_diff::assert_json_eq; -use serde_json::json; -use serde_json::Value; - -#[macro_use] mod common; - -#[actix_rt::test] -async fn search() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "exercitation" - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let hits: Vec = hits.iter().cloned().take(3).collect(); - assert_json_eq!(expected.clone(), serde_json::to_value(hits).unwrap(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_no_params() { - let mut server = common::Server::test_server().await; - - let query = json! ({}); - - // an empty search should return the 20 first indexed document - let dataset: Vec = serde_json::from_slice(include_bytes!("assets/test_set.json")).unwrap(); - let expected: Vec = dataset.into_iter().take(20).collect(); - let expected: Value = serde_json::to_value(expected).unwrap(); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_in_unexisting_index() { - let mut server = common::Server::with_uid("test"); - - let query = json! ({ - "q": "exercitation" - }); - - let expected = json! ({ - "message": "Index test not found", - "errorCode": "index_not_found", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#index_not_found" - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(404, status_code); - assert_json_eq!(expected.clone(), response.clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_unexpected_params() { - - let query = json! ({"lol": "unexpected"}); - - let expected = "unknown field `lol`, expected one of `q`, `offset`, `limit`, `attributesToRetrieve`, `attributesToCrop`, `cropLength`, `attributesToHighlight`, `filters`, `matches`, `facetFilters`, `facetsDistribution` at line 1 column 6"; - - let post_query = serde_json::from_str::(&query.to_string()); - assert!(post_query.is_err()); - assert_eq!(expected, post_query.err().unwrap().to_string()); - - let get_query: Result = serde_json::from_str(&query.to_string()); - assert!(get_query.is_err()); - assert_eq!(expected, get_query.err().unwrap().to_string()); -} - -#[actix_rt::test] -async fn search_with_limit() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "exercitation", - "limit": 3 - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_offset() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 3, - "offset": 1 - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_wildcard() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["*"] - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_1() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["name"] - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_matches() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "matches": true - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_matchesInfo": { - "name": [ - { - "start": 0, - "length": 6 - } - ], - "email": [ - { - "start": 0, - "length": 6 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_crop() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToCrop": ["about"], - "cropLength": 20 - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_retrieve() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","color","gender"], - }); - - let expected = json!([ - { - "name": "Cherry Orr", - "age": 27, - "color": "Green", - "gender": "female" - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_retrieve_wildcard() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["*"], - }); - - let expected = json!([ - { - "id": 1, - "isActive": true, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ] - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_filter() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "gender='male'" - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - }, - { - "id": 66, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - } - ]); - - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "name='Lucas Hess'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 2, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ], - "isActive": true - }, - { - "id": 75, - "balance": "$1,913.42", - "picture": "http://placehold.it/32x32", - "age": 24, - "color": "Green", - "name": "Emma Jacobs", - "gender": "female", - "email": "emmajacobs@chorizon.com", - "phone": "+1 (899) 554-3847", - "address": "173 Tapscott Street, Esmont, Maine, 7450", - "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", - "registered": "2019-03-29T06:24:13 -01:00", - "latitude": -35.53722, - "longitude": 155.703874, - "tags": [], - "isActive": false - } - ]); - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "gender='female' AND (name='Patricia Goff' OR name='Emma Jacobs')" - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 30, - "balance": "$2,021.11", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Stacy Espinoza", - "gender": "female", - "email": "stacyespinoza@chorizon.com", - "phone": "+1 (999) 487-3253", - "address": "931 Alabama Avenue, Bangor, Alaska, 8215", - "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", - "registered": "2014-07-16T06:15:53 -02:00", - "latitude": 41.560197, - "longitude": 177.697, - "tags": [ - "new issue", - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 31, - "balance": "$3,609.82", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Vilma Garza", - "gender": "female", - "email": "vilmagarza@chorizon.com", - "phone": "+1 (944) 585-2021", - "address": "565 Tech Place, Sedley, Puerto Rico, 858", - "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", - "registered": "2017-06-30T07:43:52 -02:00", - "latitude": -12.574889, - "longitude": -54.771186, - "tags": [ - "new issue", - "wontfix", - "wontfix" - ], - "isActive": false - }, - { - "id": 2, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "limit": 3, - "filters": "gender='female' AND (name='Patricia Goff' OR age > 30)" - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - }, - { - "id": 66, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "limit": 3, - "filters": "NOT gender = 'female' AND age > 30" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 11, - "balance": "$1,351.43", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Evans Wagner", - "gender": "male", - "email": "evanswagner@chorizon.com", - "phone": "+1 (889) 496-2332", - "address": "118 Monaco Place, Lutsen, Delaware, 6209", - "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", - "registered": "2016-10-27T01:26:31 -02:00", - "latitude": -77.673222, - "longitude": -142.657214, - "tags": [ - "good first issue", - "good first issue" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "filters": "NOT gender = 'female' AND name='Evans Wagner'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_highlight_and_matches() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["name","email"], - "matches": true, - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - "_matchesInfo": { - "email": [ - { - "start": 0, - "length": 6 - } - ], - "name": [ - { - "start": 0, - "length": 6 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_highlight_and_matches_and_crop() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exerciatation", - "limit": 1, - "attributesToCrop": ["about"], - "cropLength": 20, - "attributesToHighlight": ["about"], - "matches": true, - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - "_matchesInfo": { - "about": [ - { - "start": 0, - "length": 12 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","gender","email"], - "attributesToHighlight": ["name"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry Orr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_2() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToRetrieve": ["name","age","gender"], - "attributesToCrop": ["about"], - "cropLength": 20, - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "_formatted": { - "about": "Exercitation officia" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_3() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToRetrieve": ["name","age","gender"], - "attributesToCrop": ["about:20"], - }); - - let expected = json!( [ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "_formatted": { - "about": "Exercitation officia" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_4() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["name:0","email:6"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry", - "email": "cherryorr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_5() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["*","email:6"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry Orr", - "email": "cherryorr", - "age": 27, - "gender": "female" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_6() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["*","email:10"], - "attributesToHighlight": ["name"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_7() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","gender","email"], - "attributesToCrop": ["*","email:6"], - "attributesToHighlight": ["*"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_8() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender","address"], - "attributesToCrop": ["*","email:6"], - "attributesToHighlight": ["*","address"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn test_faceted_search_valid() { - // set facetting attributes before adding documents - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - - let body = json!({ - "attributesForFaceting": ["color"] - }); - server.update_all_settings(body).await; - - let dataset = include_bytes!("assets/test_set.json"); - let body: Value = serde_json::from_slice(dataset).unwrap(); - server.add_or_update_multiple_documents(body).await; - - // simple tests on attributes with string value - - let query = json!({ - "q": "a", - "facetFilters": ["color:green"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "Green")); - }); - - let query = json!({ - "q": "a", - "facetFilters": [["color:blue"]] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - let query = json!({ - "q": "a", - "facetFilters": ["color:Blue"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - // test on arrays: ["tags:bug"] - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "q": "a", - "facetFilters": ["tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); - }); - - // test and: ["color:blue", "tags:bug"] - let query = json!({ - "q": "a", - "facetFilters": ["color:blue", "tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("color") - .unwrap() == "blue" - && value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); - }); - - // test or: [["color:blue", "color:green"]] - let query = json!({ - "q": "a", - "facetFilters": [["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| - value - .get("color") - .unwrap() == "blue" - || value - .get("color") - .unwrap() == "Green")); - }); - // test and-or: ["tags:bug", ["color:blue", "color:green"]] - let query = json!({ - "q": "a", - "facetFilters": ["tags:bug", ["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| - value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())) - && (value - .get("color") - .unwrap() == "blue" - || value - .get("color") - .unwrap() == "Green"))); - - }); -} - -#[actix_rt::test] -async fn test_faceted_search_invalid() { - let mut server = common::Server::test_server().await; - - //no faceted attributes set - let query = json!({ - "q": "a", - "facetFilters": ["color:blue"] - }); - - test_post_get_search!(server, query, |response, status_code| { - - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // empty arrays are error - // [] - let query = json!({ - "q": "a", - "facetFilters": [] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - // [[]] - let query = json!({ - "q": "a", - "facetFilters": [[]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // ["color:green", []] - let query = json!({ - "q": "a", - "facetFilters": ["color:green", []] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // too much depth - // [[[]]] - let query = json!({ - "q": "a", - "facetFilters": [[[]]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // [["color:green", ["color:blue"]]] - let query = json!({ - "q": "a", - "facetFilters": [["color:green", ["color:blue"]]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // "color:green" - let query = json!({ - "q": "a", - "facetFilters": "color:green" - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); -} - -#[actix_rt::test] -async fn test_facet_count() { - let mut server = common::Server::test_server().await; - - // test without facet distribution - let query = json!({ - "q": "a", - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert!(response.get("exhaustiveFacetsCount").is_none()); - assert!(response.get("facetsDistribution").is_none()); - }); - - // test no facets set, search on color - let query = json!({ - "q": "a", - "facetsDistribution": ["color"] - }); - test_post_get_search!(server, query.clone(), |_response, status_code|{ - assert_eq!(status_code, 400); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // same as before, but now facets are set: - test_post_get_search!(server, query, |response, _status_code|{ - println!("{}", response); - assert!(response.get("exhaustiveFacetsCount").is_some()); - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 1); - // assert that case is preserved - assert!(response["facetsDistribution"] - .as_object() - .unwrap()["color"] - .as_object() - .unwrap() - .get("Green") - .is_some()); - }); - // searching on color and tags - let query = json!({ - "q": "a", - "facetsDistribution": ["color", "tags"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - let facets = response.get("facetsDistribution").unwrap().as_object().unwrap(); - assert_eq!(facets.values().count(), 2); - assert_ne!(!facets.get("color").unwrap().as_object().unwrap().values().count(), 0); - assert_ne!(!facets.get("tags").unwrap().as_object().unwrap().values().count(), 0); - }); - // wildcard - let query = json!({ - "q": "a", - "facetsDistribution": ["*"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); - }); - // wildcard with other attributes: - let query = json!({ - "q": "a", - "facetsDistribution": ["color", "*"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); - }); - - // empty facet list - let query = json!({ - "q": "a", - "facetsDistribution": [] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 0); - }); - - // attr not set as facet passed: - let query = json!({ - "q": "a", - "facetsDistribution": ["gender"] - }); - test_post_get_search!(server, query, |_response, status_code|{ - assert_eq!(status_code, 400); - }); - -} - -#[actix_rt::test] -#[should_panic] -async fn test_bad_facet_distribution() { - let mut server = common::Server::test_server().await; - // string instead of array: - let query = json!({ - "q": "a", - "facetsDistribution": "color" - }); - test_post_get_search!(server, query, |_response, _status_code| {}); - - // invalid value in array: - let query = json!({ - "q": "a", - "facetsDistribution": ["color", true] - }); - test_post_get_search!(server, query, |_response, _status_code| {}); -} - -#[actix_rt::test] -async fn highlight_cropped_text() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let doc = json!([ - { - "id": 1, - "body": r##"well, it may not work like that, try the following: -1. insert your trip -2. google your `searchQuery` -3. find a solution -> say hello"## - } - ]); - server.add_or_replace_multiple_documents(doc).await; - - // tests from #680 - //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=30"; - let query = json!({ - "q": "insert", - "attributesToHighlight": ["*"], - "attributesToCrop": ["body"], - "cropLength": 30, - }); - let expected_response = "that, try the following: \n1. insert your trip\n2. google your"; - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .get(0) - .unwrap() - .as_object() - .unwrap() - .get("_formatted") - .unwrap() - .as_object() - .unwrap() - .get("body") - .unwrap() - , &Value::String(expected_response.to_owned())); - }); - - //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=80"; - let query = json!({ - "q": "insert", - "attributesToHighlight": ["*"], - "attributesToCrop": ["body"], - "cropLength": 80, - }); - let expected_response = "well, it may not work like that, try the following: \n1. insert your trip\n2. google your `searchQuery`\n3. find a solution \n> say hello"; - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .get(0) - .unwrap() - .as_object() - .unwrap() - .get("_formatted") - .unwrap() - .as_object() - .unwrap() - .get("body") - .unwrap() - , &Value::String(expected_response.to_owned())); - }); -} - -#[actix_rt::test] -async fn well_formated_error_with_bad_request_params() { - let mut server = common::Server::with_uid("test"); - let query = "foo=bar"; - let (response, _status_code) = server.search_get(query).await; - assert!(response.get("message").is_some()); - assert!(response.get("errorCode").is_some()); - assert!(response.get("errorType").is_some()); - assert!(response.get("errorLink").is_some()); -} - - -#[actix_rt::test] -async fn update_documents_with_facet_distribution() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let settings = json!({ - "attributesForFaceting": ["genre"], - "displayedAttributes": ["genre"], - "searchableAttributes": ["genre"] - }); - server.update_all_settings(settings).await; - let update1 = json!([ - { - "id": "1", - "type": "album", - "title": "Nevermind", - "genre": ["grunge", "alternative"] - }, - { - "id": "2", - "type": "album", - "title": "Mellon Collie and the Infinite Sadness", - "genre": ["alternative", "rock"] - }, - { - "id": "3", - "type": "album", - "title": "The Queen Is Dead", - "genre": ["indie", "rock"] - } - ]); - server.add_or_update_multiple_documents(update1).await; - let search = json!({ - "q": "album", - "facetsDistribution": ["genre"] - }); - let (response1, _) = server.search_post(search.clone()).await; - let expected_facet_distribution = json!({ - "genre": { - "grunge": 1, - "alternative": 2, - "rock": 2, - "indie": 1 - } - }); - assert_json_eq!(expected_facet_distribution.clone(), response1["facetsDistribution"].clone()); - - let update2 = json!([ - { - "id": "3", - "title": "The Queen Is Very Dead" - } - ]); - server.add_or_update_multiple_documents(update2).await; - let (response2, _) = server.search_post(search).await; - assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone()); -} - -#[actix_rt::test] -async fn test_filter_nb_hits_search_normal() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let documents = json!([ - { - "id": 1, - "content": "a", - "color": "green", - "size": 1, - }, - { - "id": 2, - "content": "a", - "color": "green", - "size": 2, - }, - { - "id": 3, - "content": "a", - "color": "blue", - "size": 3, - }, - ]); - - server.add_or_update_multiple_documents(documents).await; - let (response, _) = server.search_post(json!({"q": "a"})).await; - assert_eq!(response["nbHits"], 3); - - let (response, _) = server.search_post(json!({"q": "a", "filters": "size = 1"})).await; - assert_eq!(response["nbHits"], 1); - - server.update_distinct_attribute(json!("color")).await; - - let (response, _) = server.search_post(json!({"q": "a"})).await; - assert_eq!(response["nbHits"], 2); - - let (response, _) = server.search_post(json!({"q": "a", "filters": "size < 3"})).await; - println!("result: {}", response); - assert_eq!(response["nbHits"], 1); -} diff --git a/tests/search_settings.rs b/tests/search_settings.rs deleted file mode 100644 index 46417498d..000000000 --- a/tests/search_settings.rs +++ /dev/null @@ -1,538 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; -use std::convert::Into; - -mod common; - -#[actix_rt::test] -async fn search_with_settings_basic() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=ea%20exercitation&limit=3"; - - let expect = json!([ - { - "balance": "$2,467.47", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700" - }, - { - "balance": "$3,344.40", - "age": 35, - "color": "blue", - "name": "Adeline Flynn", - "gender": "female", - "email": "adelineflynn@chorizon.com", - "phone": "+1 (994) 600-2840", - "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948" - }, - { - "balance": "$3,394.96", - "age": 25, - "color": "blue", - "name": "Aida Kirby", - "gender": "female", - "email": "aidakirby@chorizon.com", - "phone": "+1 (942) 532-2325", - "address": "797 Engert Avenue, Wilsonia, Idaho, 6532" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_stop_words() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": ["ea"], - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=ea%20exercitation&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_synonyms() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "application": [ - "exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=application&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_ranking_rules() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exarcitation&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - println!("{}", response["hits"].clone()); - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_searchable_attributes() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "exarcitation": [ - "exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=Carol&limit=3"; - let expect = json!([ - { - "balance": "$1,440.09", - "age": 40, - "color": "blue", - "name": "Levy Whitley", - "gender": "male", - "email": "levywhitley@chorizon.com", - "phone": "+1 (911) 458-2411", - "address": "187 Thomas Street, Hachita, North Carolina, 2989" - }, - { - "balance": "$1,977.66", - "age": 36, - "color": "brown", - "name": "Combs Stanley", - "gender": "male", - "email": "combsstanley@chorizon.com", - "phone": "+1 (827) 419-2053", - "address": "153 Beverley Road, Siglerville, South Carolina, 3666" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_displayed_attributes() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exercitation&limit=3"; - let expect = json!([ - { - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243" - }, - { - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174" - }, - { - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_searchable_attributes_2() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exercitation&limit=3"; - let expect = json!([ - { - "age": 31, - "name": "Harper Carson", - "gender": "male" - }, - { - "age": 27, - "name": "Cherry Orr", - "gender": "female" - }, - { - "age": 28, - "name": "Maureen Dale", - "gender": "female" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -// issue #798 -#[actix_rt::test] -async fn distinct_attributes_returns_name_not_id() { - let mut server = common::Server::test_server().await; - let settings = json!({ - "distinctAttribute": "color", - }); - server.update_all_settings(settings).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["distinctAttribute"], "color"); - let (response, _) = server.get_distinct_attribute().await; - assert_eq!(response, "color"); -} diff --git a/tests/settings.rs b/tests/settings.rs deleted file mode 100644 index 6b125c13a..000000000 --- a/tests/settings.rs +++ /dev/null @@ -1,523 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; -use std::convert::Into; -mod common; - -#[actix_rt::test] -async fn write_all_and_delete() { - let mut server = common::Server::test_server().await; - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all settings - - server.delete_all_settings().await; - - // 5 - Get all settings and check if they are set to default values - - let (response, _status_code) = server.get_all_settings().await; - - let expect = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - assert_json_eq!(expect, response, ordered: false); -} - -#[actix_rt::test] -async fn write_all_and_update() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Update all settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "color", - "age", - ], - "displayedAttributes": [ - "name", - "color", - "age", - "registered", - "picture", - ], - "stopWords": [], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["title"], - }); - - server.update_all_settings(body).await; - - // 5 - Get all settings and check if the content is the same of (4) - - let (response, _status_code) = server.get_all_settings().await; - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "color", - "age", - ], - "displayedAttributes": [ - "name", - "color", - "age", - "registered", - "picture", - ], - "stopWords": [], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["title"], - }); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn test_default_settings() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - - // 1 - Get all settings and compare to the previous one - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); -} - -#[actix_rt::test] -async fn test_default_settings_2() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - // 1 - Get all settings and compare to the previous one - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/516 -#[actix_rt::test] -async fn write_setting_and_update_partial() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - - // 2 - Send the settings - - let body = json!({ - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ] - }); - - server.update_all_settings(body.clone()).await; - - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - "desc(registered)", - ], - "distinctAttribute": "id", - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - }); - - server.update_all_settings(body.clone()).await; - - // 2 - Send the settings - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - "desc(registered)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn attributes_for_faceting_settings() { - let mut server = common::Server::test_server().await; - // initial attributes array should be empty - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!([])); - // add an attribute and test for its presence - let (_response, _status_code) = server.post_request_async( - "/indexes/test/settings/attributes-for-faceting", - json!(["foobar"])).await; - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!(["foobar"])); - // remove all attributes and test for emptiness - let (_response, _status_code) = server.delete_request_async( - "/indexes/test/settings/attributes-for-faceting").await; - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!([])); -} - -#[actix_rt::test] -async fn setting_ranking_rules_dont_mess_with_other_settings() { - let mut server = common::Server::test_server().await; - let body = json!({ - "rankingRules": ["asc(foobar)"] - }); - server.update_all_settings(body).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["rankingRules"].as_array().unwrap().len(), 1); - assert_eq!(response["rankingRules"].as_array().unwrap().first().unwrap().as_str().unwrap(), "asc(foobar)"); - assert!(!response["searchableAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); - assert!(!response["displayedAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); -} - -#[actix_rt::test] -async fn displayed_and_searchable_attributes_reset_to_wildcard() { - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); - - server.delete_searchable_attributes().await; - server.delete_displayed_attributes().await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); - - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); - - server.update_all_settings(json!({ "searchableAttributes": [], "displayedAttributes": [] })).await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); -} - -#[actix_rt::test] -async fn settings_that_contains_wildcard_is_wildcard() { - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color", "*"], "displayedAttributes": ["color", "*"] })).await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); -} - -#[actix_rt::test] -async fn test_displayed_attributes_field() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "age", - "email", - "gender", - "name", - "registered", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["avenue", "street"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: true); -} \ No newline at end of file diff --git a/tests/settings_ranking_rules.rs b/tests/settings_ranking_rules.rs deleted file mode 100644 index ac9a1e00c..000000000 --- a/tests/settings_ranking_rules.rs +++ /dev/null @@ -1,182 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn write_all_and_delete() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_ranking_rules().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all settings - - server.delete_ranking_rules().await; - - // 5 - Get all settings and check if they are empty - - let (response, _status_code) = server.get_ranking_rules().await; - - let expected = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ]); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn write_all_and_update() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_ranking_rules().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Update all settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - ]); - - server.update_ranking_rules(body).await; - - // 5 - Get all settings and check if the content is the same of (4) - - let (response, _status_code) = server.get_ranking_rules().await; - - let expected = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - ]); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn send_undefined_rule() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let body = json!(["typos",]); - - let (_response, status_code) = server.update_ranking_rules_sync(body).await; - assert_eq!(status_code, 400); -} - -#[actix_rt::test] -async fn send_malformed_custom_rule() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let body = json!(["dsc(truc)",]); - - let (_response, status_code) = server.update_ranking_rules_sync(body).await; - assert_eq!(status_code, 400); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/521 -#[actix_rt::test] -async fn write_custom_ranking_and_index_documents() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - // 1 - Add ranking rules with one custom ranking on a string - - let body = json!(["asc(name)", "typo"]); - - server.update_ranking_rules(body).await; - - // 2 - Add documents - - let body = json!([ - { - "id": 1, - "name": "Cherry Orr", - "color": "green" - }, - { - "id": 2, - "name": "Lucas Hess", - "color": "yellow" - } - ]); - - server.add_or_replace_multiple_documents(body).await; - - // 3 - Get the first document and compare - - let expected = json!({ - "id": 1, - "name": "Cherry Orr", - "color": "green" - }); - - let (response, status_code) = server.get_document(1).await; - assert_eq!(status_code, 200); - - assert_json_eq!(response, expected, ordered: false); -} diff --git a/tests/settings_stop_words.rs b/tests/settings_stop_words.rs deleted file mode 100644 index 3ff2e8bb7..000000000 --- a/tests/settings_stop_words.rs +++ /dev/null @@ -1,61 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn update_stop_words() { - let mut server = common::Server::test_server().await; - - // 1 - Get stop words - - let (response, _status_code) = server.get_stop_words().await; - assert_eq!(response.as_array().unwrap().is_empty(), true); - - // 2 - Update stop words - - let body = json!(["ut", "ea"]); - server.update_stop_words(body.clone()).await; - - // 3 - Get all stop words and compare to the previous one - - let (response, _status_code) = server.get_stop_words().await; - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all stop words - - server.delete_stop_words().await; - - // 5 - Get all stop words and check if they are empty - - let (response, _status_code) = server.get_stop_words().await; - assert_eq!(response.as_array().unwrap().is_empty(), true); -} - -#[actix_rt::test] -async fn add_documents_and_stop_words() { - let mut server = common::Server::test_server().await; - - // 2 - Update stop words - - let body = json!(["ad", "in"]); - server.update_stop_words(body.clone()).await; - - // 3 - Search for a document with stop words - - let (response, _status_code) = server.search_get("q=in%20exercitation").await; - assert!(!response["hits"].as_array().unwrap().is_empty()); - - // 4 - Search for documents with *only* stop words - - let (response, _status_code) = server.search_get("q=ad%20in").await; - assert!(response["hits"].as_array().unwrap().is_empty()); - - // 5 - Delete all stop words - - // server.delete_stop_words(); - - // // 6 - Search for a document with one stop word - - // assert!(!response["hits"].as_array().unwrap().is_empty()); -} diff --git a/tests/url_normalizer.rs b/tests/url_normalizer.rs deleted file mode 100644 index c2c9187ee..000000000 --- a/tests/url_normalizer.rs +++ /dev/null @@ -1,18 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn url_normalizer() { - let mut server = common::Server::with_uid("movies"); - - let (_response, status_code) = server.get_request("/version").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("//version").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("/version/").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("//version/").await; - assert_eq!(status_code, 200); -} From ccb7104dee63b631d28ad5f4356157221d4325f9 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 29 Jan 2021 19:14:23 +0100 Subject: [PATCH 027/522] add tests for IndexStore --- .../local_index_controller/index_store.rs | 155 +++++++++++++++--- src/option.rs | 18 ++ 2 files changed, 153 insertions(+), 20 deletions(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index b94d9e492..c96884155 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -17,11 +17,11 @@ use super::{UpdateMeta, UpdateResult}; type UpdateStore = super::update_store::UpdateStore; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, PartialEq)] struct IndexMeta { update_size: u64, index_size: u64, - uid: Uuid, + uuid: Uuid, } impl IndexMeta { @@ -31,8 +31,8 @@ impl IndexMeta { thread_pool: Arc, opt: &IndexerOpts, ) -> anyhow::Result<(Arc, Arc)> { - let update_path = make_update_db_path(&path, &self.uid); - let index_path = make_index_db_path(&path, &self.uid); + let update_path = make_update_db_path(&path, &self.uuid); + let index_path = make_index_db_path(&path, &self.uuid); create_dir_all(&update_path)?; create_dir_all(&index_path)?; @@ -51,7 +51,6 @@ impl IndexMeta { pub struct IndexStore { env: Env, - name_to_uid: DashMap, name_to_uid_db: Database, uid_to_index: DashMap, Arc)>, uid_to_index_db: Database>, @@ -67,7 +66,6 @@ impl IndexStore { .max_dbs(2) .open(path)?; - let name_to_uid = DashMap::new(); let uid_to_index = DashMap::new(); let name_to_uid_db = open_or_create_database(&env, Some("name_to_uid"))?; let uid_to_index_db = open_or_create_database(&env, Some("uid_to_index_db"))?; @@ -79,7 +77,6 @@ impl IndexStore { Ok(Self { env, - name_to_uid, name_to_uid_db, uid_to_index, uid_to_index_db, @@ -90,18 +87,12 @@ impl IndexStore { } fn index_uid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uid.entry(name.as_ref().to_string()) { - Entry::Vacant(entry) => { - match self.name_to_uid_db.get(txn, name.as_ref())? { - Some(bytes) => { - let uuid = Uuid::from_slice(bytes)?; - entry.insert(uuid); - Ok(Some(uuid)) - } - None => Ok(None) - } + match self.name_to_uid_db.get(txn, name.as_ref())? { + Some(bytes) => { + let uuid = Uuid::from_slice(bytes)?; + Ok(Some(uuid)) } - Entry::Occupied(entry) => Ok(Some(entry.get().clone())), + None => Ok(None) } } @@ -182,7 +173,7 @@ impl IndexStore { update_size: u64, index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { - let meta = IndexMeta { update_size, index_size, uid: uid.clone() }; + let meta = IndexMeta { update_size, index_size, uuid: uid.clone() }; self.name_to_uid_db.put(txn, name.as_ref(), uid.as_bytes())?; self.uid_to_index_db.put(txn, uid.as_bytes(), &meta)?; @@ -190,7 +181,6 @@ impl IndexStore { let path = self.env.path(); let (index, update_store) = meta.open(path, self.thread_pool.clone(), &self.opt)?; - self.name_to_uid.insert(name.as_ref().to_string(), uid); self.uid_to_index.insert(uid, (index.clone(), update_store.clone())); Ok((index, update_store)) @@ -215,3 +205,128 @@ fn make_index_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { path.push(format!("index{}", uid)); path } + +#[cfg(test)] +mod test { + use super::*; + use std::path::PathBuf; + + #[test] + fn test_make_update_db_path() { + let uuid = Uuid::new_v4(); + assert_eq!( + make_update_db_path("/home", &uuid), + PathBuf::from(format!("/home/update{}", uuid)) + ); + } + + #[test] + fn test_make_index_db_path() { + let uuid = Uuid::new_v4(); + assert_eq!( + make_index_db_path("/home", &uuid), + PathBuf::from(format!("/home/index{}", uuid)) + ); + } + + mod index_store { + use super::*; + + #[test] + fn test_index_uuid() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + + let name = "foobar"; + let txn = store.env.read_txn().unwrap(); + // name is not found if the uuid in not present in the db + assert!(store.index_uid(&txn, &name).unwrap().is_none()); + drop(txn); + + // insert an uuid in the the name_to_uuid_db: + let uuid = Uuid::new_v4(); + let mut txn = store.env.write_txn().unwrap(); + store.name_to_uid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + txn.commit().unwrap(); + + // check that the uuid is there + let txn = store.env.read_txn().unwrap(); + assert_eq!(store.index_uid(&txn, &name).unwrap(), Some(uuid)); + } + + #[test] + fn test_retrieve_index() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + let uuid = Uuid::new_v4(); + + let txn = store.env.read_txn().unwrap(); + assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); + + let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let mut txn = store.env.write_txn().unwrap(); + store.uid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + txn.commit().unwrap(); + + // the index cache should be empty + assert!(store.uid_to_index.is_empty()); + + let txn = store.env.read_txn().unwrap(); + assert!(store.retrieve_index(&txn, uuid).unwrap().is_some()); + assert_eq!(store.uid_to_index.len(), 1); + } + + #[test] + fn test_index() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + let name = "foobar"; + + assert!(store.index(&name).unwrap().is_none()); + + let uuid = Uuid::new_v4(); + let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let mut txn = store.env.write_txn().unwrap(); + store.name_to_uid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.uid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + txn.commit().unwrap(); + + assert!(store.index(&name).unwrap().is_some()); + } + + #[test] + fn test_get_or_create_index() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + let name = "foobar"; + + store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap(); + let txn = store.env.read_txn().unwrap(); + let uuid = store.name_to_uid_db.get(&txn, &name).unwrap(); + assert_eq!(store.uid_to_index.len(), 1); + assert!(uuid.is_some()); + let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); + let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + assert_eq!(store.uid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + } + + #[test] + fn test_create_index() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + let name = "foobar"; + + let update_size = 4096 * 100; + let index_size = 4096 * 100; + let uuid = Uuid::new_v4(); + let mut txn = store.env.write_txn().unwrap(); + store.create_index(&mut txn, uuid, name, update_size, index_size).unwrap(); + let uuid = store.name_to_uid_db.get(&txn, &name).unwrap(); + assert_eq!(store.uid_to_index.len(), 1); + assert!(uuid.is_some()); + let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); + let meta = IndexMeta { update_size , index_size, uuid: uuid.clone() }; + assert_eq!(store.uid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + } + } +} diff --git a/src/option.rs b/src/option.rs index f280553e1..dffa7d483 100644 --- a/src/option.rs +++ b/src/option.rs @@ -63,6 +63,24 @@ pub struct IndexerOpts { #[structopt(long)] pub indexing_jobs: Option, } + +#[cfg(test)] +impl Default for IndexerOpts { + fn default() -> Self { + Self { + log_every_n: 0, + max_nb_chunks: None, + max_memory: Byte::from_str("0Kb").unwrap(), + linked_hash_map_size: 0, + chunk_compression_type: CompressionType::None, + chunk_compression_level: None, + chunk_fusing_shrink_size: Byte::from_str("0Kb").unwrap(), + enable_chunk_fusing: false, + indexing_jobs: None, + } + } +} + const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; #[derive(Debug, Clone, StructOpt)] From 17c463ca61eaaccc9bbf106e871506a030bb878d Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Feb 2021 13:32:21 +0100 Subject: [PATCH 028/522] remove unused deps --- Cargo.lock | 161 ++++------------------------------------------------- Cargo.toml | 5 -- 2 files changed, 11 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4471f5127..9240548d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,15 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -dependencies = [ - "lazy_static", - "regex", -] - [[package]] name = "actix-codec" version = "0.3.0" @@ -41,7 +31,7 @@ dependencies = [ "futures-util", "http", "log", - "rustls 0.18.1", + "rustls", "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", @@ -217,10 +207,10 @@ dependencies = [ "actix-service", "actix-utils", "futures-util", - "rustls 0.18.1", + "rustls", "tokio-rustls", "webpki", - "webpki-roots 0.20.0", + "webpki-roots", ] [[package]] @@ -273,7 +263,7 @@ dependencies = [ "mime", "pin-project 1.0.2", "regex", - "rustls 0.18.1", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -342,7 +332,7 @@ checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479" [[package]] name = "assert-json-diff" version = "1.0.1" -source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +source = "git+https://github.com/qdequele/assert-json-diff#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" dependencies = [ "serde", "serde_json", @@ -408,7 +398,7 @@ dependencies = [ "mime", "percent-encoding", "rand 0.7.3", - "rustls 0.18.1", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -634,12 +624,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "chunked_transfer" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca" - [[package]] name = "clap" version = "2.33.3" @@ -1365,7 +1349,7 @@ dependencies = [ "futures-util", "hyper", "log", - "rustls 0.18.1", + "rustls", "tokio", "tokio-rustls", "webpki", @@ -1670,16 +1654,14 @@ dependencies = [ "mime", "obkv", "once_cell", - "ouroboros", "page_size", "rand 0.7.3", "rayon", "regex", - "rustls 0.18.1", + "rustls", "sentry", "serde", "serde_json", - "serde_qs", "serde_url_params", "sha2", "siphasher", @@ -1689,11 +1671,8 @@ dependencies = [ "tempdir", "tempfile", "tokio", - "ureq", "uuid", "vergen", - "walkdir", - "whoami", ] [[package]] @@ -1974,29 +1953,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ouroboros" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069fb33e127cabdc8ad6a287eed9719b85c612d36199777f6dc41ad91f7be41a" -dependencies = [ - "ouroboros_macro", - "stable_deref_trait", -] - -[[package]] -name = "ouroboros_macro" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad938cc920f299d6dce91e43d3ce316e785f4aa4bc4243555634dc2967098fc6" -dependencies = [ - "Inflector", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "page_size" version = "0.4.2" @@ -2242,15 +2198,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "qstring" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d464fae65fff2680baf48019211ce37aaec0c78e9264c84a3e484717f965104e" -dependencies = [ - "percent-encoding", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -2453,7 +2400,7 @@ dependencies = [ "mime_guess", "percent-encoding", "pin-project-lite 0.2.0", - "rustls 0.18.1", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -2463,7 +2410,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.20.0", + "webpki-roots", "winreg 0.7.0", ] @@ -2535,34 +2482,12 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" -dependencies = [ - "base64 0.13.0", - "log", - "ring", - "sct", - "webpki", -] - [[package]] name = "ryu" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.1.0" @@ -2674,17 +2599,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_qs" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5cb0f0564a84554436c4ceff5c896308d4e09d0eb4bd0215b8f698f88084601" -dependencies = [ - "percent-encoding", - "serde", - "thiserror", -] - [[package]] name = "serde_url_params" version = "0.2.0" @@ -2827,12 +2741,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "standback" version = "0.2.13" @@ -3175,7 +3083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", - "rustls 0.18.1", + "rustls", "tokio", "webpki", ] @@ -3349,23 +3257,6 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" -[[package]] -name = "ureq" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "294b85ef5dbc3670a72e82a89971608a1fcc4ed5c7c5a2895230d31a95f0569b" -dependencies = [ - "base64 0.13.0", - "chunked_transfer", - "log", - "once_cell", - "qstring", - "rustls 0.19.0", - "url", - "webpki", - "webpki-roots 0.21.0", -] - [[package]] name = "url" version = "2.2.0" @@ -3417,17 +3308,6 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" -[[package]] -name = "walkdir" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" -dependencies = [ - "same-file", - "winapi 0.3.9", - "winapi-util", -] - [[package]] name = "want" version = "0.3.0" @@ -3547,15 +3427,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "webpki-roots" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" -dependencies = [ - "webpki", -] - [[package]] name = "whatlang" version = "0.9.0" @@ -3565,16 +3436,6 @@ dependencies = [ "hashbrown 0.7.2", ] -[[package]] -name = "whoami" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e35495e7faf4c657051a8e9725d9c37ac57879e915be3ed55bb401af84382035" -dependencies = [ - "wasm-bindgen", - "web-sys", -] - [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index f68414beb..1d7590d47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,6 @@ regex = "1.4.2" rustls = "0.18" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.59", features = ["preserve_order"] } -serde_qs = "0.8.1" sha2 = "0.9.1" siphasher = "0.3.2" slice-group-by = "0.2.6" @@ -55,13 +54,9 @@ structopt = "0.3.20" tar = "0.4.29" tempfile = "3.1.0" tokio = { version = "0.2", features = ["full"] } -ureq = { version = "1.5.1", default-features = false, features = ["tls"] } -walkdir = "2.3.1" -whoami = "1.0.0" dashmap = "4.0.2" page_size = "0.4.2" obkv = "0.1.1" -ouroboros = "0.8.0" uuid = "0.8.2" itertools = "0.10.0" From 9af0a08122e3558d90a0ac32fff63dcc7ba1a912 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Feb 2021 19:51:47 +0100 Subject: [PATCH 029/522] post review fixes --- Cargo.lock | 113 ++++++++++-- Cargo.toml | 1 + src/data/mod.rs | 8 +- src/data/search.rs | 24 ++- src/data/updates.rs | 27 ++- .../local_index_controller/index_store.rs | 173 +++++++++--------- .../local_index_controller/mod.rs | 9 +- .../local_index_controller/update_store.rs | 10 - src/index_controller/mod.rs | 25 +-- src/lib.rs | 1 - 10 files changed, 233 insertions(+), 158 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9240548d9..f036664f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -808,6 +808,12 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + [[package]] name = "digest" version = "0.8.1" @@ -832,6 +838,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "downcast" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" + [[package]] name = "either" version = "1.6.1" @@ -937,6 +949,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -953,6 +974,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" + [[package]] name = "fs_extra" version = "1.2.0" @@ -1612,7 +1639,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.18.0" +version = "0.19.0" dependencies = [ "actix-http", ] @@ -1652,6 +1679,7 @@ dependencies = [ "memmap", "milli", "mime", + "mockall", "obkv", "once_cell", "page_size", @@ -1724,7 +1752,6 @@ dependencies = [ "bstr", "byte-unit", "byteorder", - "chrono", "crossbeam-channel", "csv", "either", @@ -1754,7 +1781,6 @@ dependencies = [ "roaring", "serde", "serde_json", - "serde_millis", "slice-group-by", "smallstr", "smallvec", @@ -1854,6 +1880,33 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "mockall" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "619634fd9149c4a06e66d8fd9256e85326d8eeee75abee4565ff76c92e4edfe0" +dependencies = [ + "cfg-if 1.0.0", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83714c95dbf4c24202f0f1b208f0f248e6bd65abfa8989303611a71c0f781548" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "near-proximity" version = "0.1.0" @@ -1885,6 +1938,12 @@ dependencies = [ "libc", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "num-integer" version = "0.1.44" @@ -2153,6 +2212,35 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "predicates" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" + +[[package]] +name = "predicates-tree" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" +dependencies = [ + "predicates-core", + "treeline", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2447,9 +2535,9 @@ checksum = "21215c1b9d8f7832b433255bd9eea3e2779aa55b21b2f8e13aad62c74749b237" [[package]] name = "roaring" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb550891a98438463978260676feef06c12bfb0eb0b05e191f888fb785cc9374" +checksum = "4d60b41c8f25d07cecab125cb46ebbf234fc055effc61ca2392a3ef4f9422304" dependencies = [ "byteorder", ] @@ -2590,15 +2678,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_millis" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e2dc780ca5ee2c369d1d01d100270203c4ff923d2a4264812d723766434d00" -dependencies = [ - "serde", -] - [[package]] name = "serde_url_params" version = "0.2.0" @@ -3139,6 +3218,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "treeline" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" + [[package]] name = "trust-dns-proto" version = "0.19.6" diff --git a/Cargo.toml b/Cargo.toml index 1d7590d47..ad2d034ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ serde_url_params = "0.2.0" tempdir = "0.3.7" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } tokio = { version = "0.2", features = ["macros", "time"] } +mockall = "0.9.0" [features] default = ["sentry"] diff --git a/src/data/mod.rs b/src/data/mod.rs index 175aedba5..de24d0a06 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,14 +3,14 @@ mod updates; pub use search::{SearchQuery, SearchResult}; +use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; -use std::fs::create_dir_all; use sha2::Digest; -use crate::{option::Opt, index_controller::Settings}; use crate::index_controller::{IndexController, LocalIndexController}; +use crate::{option::Opt, index_controller::Settings}; #[derive(Clone)] pub struct Data { @@ -67,7 +67,7 @@ impl Data { options.max_mdb_size.get_bytes(), options.max_udb_size.get_bytes(), )?; - let indexes = Arc::new(index_controller); + let index_controller = Arc::new(index_controller); let mut api_keys = ApiKeys { master: options.clone().master_key, @@ -77,7 +77,7 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { index_controller: indexes, options, api_keys }; + let inner = DataInner { index_controller, options, api_keys }; let inner = Arc::new(inner); Ok(Data { inner }) diff --git a/src/data/search.rs b/src/data/search.rs index 246e3bdac..2e05988aa 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -2,11 +2,11 @@ use std::collections::HashSet; use std::mem; use std::time::Instant; -use serde_json::{Value, Map}; -use serde::{Deserialize, Serialize}; -use milli::{Index, obkv_to_json, FacetCondition}; -use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; use anyhow::bail; +use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use milli::{Index, obkv_to_json, FacetCondition}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, Map}; use crate::index_controller::IndexController; use super::Data; @@ -37,7 +37,6 @@ pub struct SearchQuery { impl SearchQuery { pub fn perform(&self, index: impl AsRef) -> anyhow::Result{ let index = index.as_ref(); - let before_search = Instant::now(); let rtxn = index.read_txn().unwrap(); @@ -47,6 +46,9 @@ impl SearchQuery { search.query(query); } + search.limit(self.limit); + search.offset(self.offset.unwrap_or_default()); + if let Some(ref condition) = self.facet_condition { if !condition.trim().is_empty() { let condition = FacetCondition::from_str(&rtxn, &index, &condition).unwrap(); @@ -54,11 +56,7 @@ impl SearchQuery { } } - if let Some(offset) = self.offset { - search.offset(offset); - } - - let milli::SearchResult { documents_ids, found_words, nb_hits, limit, } = search.execute()?; + let milli::SearchResult { documents_ids, found_words, candidates } = search.execute()?; let mut documents = Vec::new(); let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); @@ -81,9 +79,9 @@ impl SearchQuery { Ok(SearchResult { hits: documents, - nb_hits, + nb_hits: candidates.len(), query: self.q.clone().unwrap_or_default(), - limit, + limit: self.limit, offset: self.offset.unwrap_or_default(), processing_time_ms: before_search.elapsed().as_millis(), }) @@ -94,7 +92,7 @@ impl SearchQuery { #[serde(rename_all = "camelCase")] pub struct SearchResult { hits: Vec>, - nb_hits: usize, + nb_hits: u64, query: String, limit: usize, offset: usize, diff --git a/src/data/updates.rs b/src/data/updates.rs index 194ec346b..27fc6537e 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -6,21 +6,20 @@ use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; use super::Data; -use crate::index_controller::{IndexController, Settings, UpdateResult, UpdateMeta}; -use crate::index_controller::updates::UpdateStatus; +use crate::index_controller::{IndexController, Settings}; +use crate::index_controller::UpdateStatus; impl Data { - pub async fn add_documents( + pub async fn add_documents( &self, - index: S, + index: impl AsRef + Send + Sync + 'static, method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, - ) -> anyhow::Result> + ) -> anyhow::Result where B: Deref, E: std::error::Error + Send + Sync + 'static, - S: AsRef + Send + Sync + 'static, { let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; let file = tokio::fs::File::from_std(file?); @@ -38,26 +37,26 @@ impl Data { let mmap = unsafe { memmap::Mmap::map(&file)? }; let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move ||index_controller.add_documents(index, method, format, &mmap[..])).await??; + let update = tokio::task::spawn_blocking(move || index_controller.add_documents(index, method, format, &mmap[..])).await??; Ok(update.into()) } - pub async fn update_settings + Send + Sync + 'static>( + pub async fn update_settings( &self, - index: S, + index: impl AsRef + Send + Sync + 'static, settings: Settings - ) -> anyhow::Result> { - let indexes = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??; + ) -> anyhow::Result { + let index_controller = self.index_controller.clone(); + let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; Ok(update.into()) } #[inline] - pub fn get_update_status>(&self, index: S, uid: u64) -> anyhow::Result>> { + pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { self.index_controller.update_status(index, uid) } - pub fn get_updates_status(&self, index: &str) -> anyhow::Result>> { + pub fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { self.index_controller.all_update_status(index) } } diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index c96884155..16df83d2c 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -1,15 +1,13 @@ +use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; -use std::fs::create_dir_all; use std::sync::Arc; -use dashmap::DashMap; -use dashmap::mapref::entry::Entry; +use dashmap::{DashMap, mapref::entry::Entry}; use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; use milli::Index; use rayon::ThreadPool; -use uuid::Uuid; use serde::{Serialize, Deserialize}; -use log::warn; +use uuid::Uuid; use crate::option::IndexerOpts; use super::update_handler::UpdateHandler; @@ -29,7 +27,7 @@ impl IndexMeta { &self, path: impl AsRef, thread_pool: Arc, - opt: &IndexerOpts, + indexer_options: &IndexerOpts, ) -> anyhow::Result<(Arc, Arc)> { let update_path = make_update_db_path(&path, &self.uuid); let index_path = make_index_db_path(&path, &self.uuid); @@ -43,24 +41,25 @@ impl IndexMeta { let mut options = EnvOpenOptions::new(); options.map_size(self.update_size as usize); - let handler = UpdateHandler::new(opt, index.clone(), thread_pool)?; + let handler = UpdateHandler::new(indexer_options, index.clone(), thread_pool)?; let update_store = UpdateStore::open(options, update_path, handler)?; + Ok((index, update_store)) } } pub struct IndexStore { env: Env, - name_to_uid_db: Database, - uid_to_index: DashMap, Arc)>, - uid_to_index_db: Database>, + name_to_uuid_db: Database, + uuid_to_index: DashMap, Arc)>, + uuid_to_index_db: Database>, thread_pool: Arc, - opt: IndexerOpts, + indexer_options: IndexerOpts, } impl IndexStore { - pub fn new(path: impl AsRef, opt: IndexerOpts) -> anyhow::Result { + pub fn new(path: impl AsRef, indexer_options: IndexerOpts) -> anyhow::Result { let env = EnvOpenOptions::new() .map_size(4096 * 100) .max_dbs(2) @@ -71,23 +70,23 @@ impl IndexStore { let uid_to_index_db = open_or_create_database(&env, Some("uid_to_index_db"))?; let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(opt.indexing_jobs.unwrap_or(0)) + .num_threads(indexer_options.indexing_jobs.unwrap_or(0)) .build()?; let thread_pool = Arc::new(thread_pool); Ok(Self { env, - name_to_uid_db, - uid_to_index, - uid_to_index_db, + name_to_uuid_db: name_to_uid_db, + uuid_to_index: uid_to_index, + uuid_to_index_db: uid_to_index_db, thread_pool, - opt, + indexer_options, }) } - fn index_uid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uid_db.get(txn, name.as_ref())? { + fn index_uuid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { + match self.name_to_uuid_db.get(txn, name.as_ref())? { Some(bytes) => { let uuid = Uuid::from_slice(bytes)?; Ok(Some(uuid)) @@ -97,12 +96,12 @@ impl IndexStore { } fn retrieve_index(&self, txn: &RoTxn, uid: Uuid) -> anyhow::Result, Arc)>> { - match self.uid_to_index.entry(uid.clone()) { + match self.uuid_to_index.entry(uid.clone()) { Entry::Vacant(entry) => { - match self.uid_to_index_db.get(txn, uid.as_bytes())? { + match self.uuid_to_index_db.get(txn, uid.as_bytes())? { Some(meta) => { let path = self.env.path(); - let (index, updates) = meta.open(path, self.thread_pool.clone(), &self.opt)?; + let (index, updates) = meta.open(path, self.thread_pool.clone(), &self.indexer_options)?; entry.insert((index.clone(), updates.clone())); Ok(Some((index, updates))) }, @@ -117,7 +116,7 @@ impl IndexStore { } fn _get_index(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { - match self.index_uid(&txn, name)? { + match self.index_uuid(&txn, name)? { Some(uid) => self.retrieve_index(&txn, uid), None => Ok(None), } @@ -129,59 +128,61 @@ impl IndexStore { } pub fn get_or_create_index( - &self, name: impl AsRef, - update_size: u64, - index_size: u64, - ) -> anyhow::Result<(Arc, Arc)> { - let mut txn = self.env.write_txn()?; - match self._get_index(&txn, name.as_ref())? { - Some(res) => Ok(res), - None => { - let uid = Uuid::new_v4(); - // TODO: clean in case of error - let result = self.create_index(&mut txn, uid, name, update_size, index_size); - match result { - Ok((index, update_store)) => { - match txn.commit() { - Ok(_) => Ok((index, update_store)), - Err(e) => { - self.clean_uid(&uid); - Err(anyhow::anyhow!("error creating index: {}", e)) - } - } - } - Err(e) => { - self.clean_uid(&uid); - Err(e) - } - } - }, - } - } - - /// removes all data acociated with an index Uuid. This is called when index creation failed - /// and outstanding files and data need to be cleaned. - fn clean_uid(&self, _uid: &Uuid) { - // TODO! - warn!("creating cleanup is not yet implemented"); - } - - fn create_index( &self, - txn: &mut RwTxn, - uid: Uuid, + &self, name: impl AsRef, update_size: u64, index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { - let meta = IndexMeta { update_size, index_size, uuid: uid.clone() }; + let mut txn = self.env.write_txn()?; + match self._get_index(&txn, name.as_ref())? { + Some(res) => Ok(res), + None => { + let uuid = Uuid::new_v4(); + let result = self.create_index(&mut txn, uuid, name, update_size, index_size)?; + // If we fail to commit the transaction, we must delete the database from the + // file-system. + if let Err(e) = txn.commit() { + self.clean_db(uuid); + return Err(e)?; + } + Ok(result) + }, + } + } - self.name_to_uid_db.put(txn, name.as_ref(), uid.as_bytes())?; - self.uid_to_index_db.put(txn, uid.as_bytes(), &meta)?; + // Remove all the files and data associated with a db uuid. + fn clean_db(&self, uuid: Uuid) { + let update_db_path = make_update_db_path(self.env.path(), &uuid); + let index_db_path = make_index_db_path(self.env.path(), &uuid); + + remove_dir_all(update_db_path).expect("Failed to clean database"); + remove_dir_all(index_db_path).expect("Failed to clean database"); + + self.uuid_to_index.remove(&uuid); + } + + fn create_index( &self, + txn: &mut RwTxn, + uuid: Uuid, + name: impl AsRef, + update_size: u64, + index_size: u64, + ) -> anyhow::Result<(Arc, Arc)> { + let meta = IndexMeta { update_size, index_size, uuid: uuid.clone() }; + + self.name_to_uuid_db.put(txn, name.as_ref(), uuid.as_bytes())?; + self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?; let path = self.env.path(); - let (index, update_store) = meta.open(path, self.thread_pool.clone(), &self.opt)?; + let (index, update_store) = match meta.open(path, self.thread_pool.clone(), &self.indexer_options) { + Ok(res) => res, + Err(e) => { + self.clean_db(uuid); + return Err(e) + } + }; - self.uid_to_index.insert(uid, (index.clone(), update_store.clone())); + self.uuid_to_index.insert(uuid, (index.clone(), update_store.clone())); Ok((index, update_store)) } @@ -194,15 +195,15 @@ fn open_or_create_database(env: &Env, name: Option<&str> } } -fn make_update_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { +fn make_update_db_path(path: impl AsRef, uuid: &Uuid) -> PathBuf { let mut path = path.as_ref().to_path_buf(); - path.push(format!("update{}", uid)); + path.push(format!("update{}", uuid)); path } -fn make_index_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { +fn make_index_db_path(path: impl AsRef, uuid: &Uuid) -> PathBuf { let mut path = path.as_ref().to_path_buf(); - path.push(format!("index{}", uid)); + path.push(format!("index{}", uuid)); path } @@ -240,18 +241,18 @@ mod test { let name = "foobar"; let txn = store.env.read_txn().unwrap(); // name is not found if the uuid in not present in the db - assert!(store.index_uid(&txn, &name).unwrap().is_none()); + assert!(store.index_uuid(&txn, &name).unwrap().is_none()); drop(txn); // insert an uuid in the the name_to_uuid_db: let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.name_to_uid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.name_to_uuid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); txn.commit().unwrap(); // check that the uuid is there let txn = store.env.read_txn().unwrap(); - assert_eq!(store.index_uid(&txn, &name).unwrap(), Some(uuid)); + assert_eq!(store.index_uuid(&txn, &name).unwrap(), Some(uuid)); } #[test] @@ -265,15 +266,15 @@ mod test { let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; let mut txn = store.env.write_txn().unwrap(); - store.uid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); // the index cache should be empty - assert!(store.uid_to_index.is_empty()); + assert!(store.uuid_to_index.is_empty()); let txn = store.env.read_txn().unwrap(); assert!(store.retrieve_index(&txn, uuid).unwrap().is_some()); - assert_eq!(store.uid_to_index.len(), 1); + assert_eq!(store.uuid_to_index.len(), 1); } #[test] @@ -287,8 +288,8 @@ mod test { let uuid = Uuid::new_v4(); let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; let mut txn = store.env.write_txn().unwrap(); - store.name_to_uid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); - store.uid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + store.name_to_uuid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); assert!(store.index(&name).unwrap().is_some()); @@ -302,12 +303,12 @@ mod test { store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap(); let txn = store.env.read_txn().unwrap(); - let uuid = store.name_to_uid_db.get(&txn, &name).unwrap(); - assert_eq!(store.uid_to_index.len(), 1); + let uuid = store.name_to_uuid_db.get(&txn, &name).unwrap(); + assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; - assert_eq!(store.uid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); } #[test] @@ -321,12 +322,12 @@ mod test { let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); store.create_index(&mut txn, uuid, name, update_size, index_size).unwrap(); - let uuid = store.name_to_uid_db.get(&txn, &name).unwrap(); - assert_eq!(store.uid_to_index.len(), 1); + let uuid = store.name_to_uuid_db.get(&txn, &name).unwrap(); + assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); let meta = IndexMeta { update_size , index_size, uuid: uuid.clone() }; - assert_eq!(store.uid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); } } } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index debc15a1a..b59eb2a99 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -2,16 +2,15 @@ mod update_store; mod index_store; mod update_handler; -use index_store::IndexStore; - use std::path::Path; use std::sync::Arc; -use milli::Index; use anyhow::bail; use itertools::Itertools; +use milli::Index; use crate::option::IndexerOpts; +use index_store::IndexStore; use super::IndexController; use super::updates::UpdateStatus; use super::{UpdateMeta, UpdateResult}; @@ -84,7 +83,7 @@ impl IndexController for LocalIndexController { } fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>> { - match self.indexes.index(index)? { + match self.indexes.index(&index)? { Some((_, update_store)) => { let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { Ok(processing @@ -99,7 +98,7 @@ impl IndexController for LocalIndexController { })?; Ok(updates) } - None => Ok(Vec::new()) + None => bail!("index {} doesn't exist.", index.as_ref()), } } diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs index 9a17ec00f..d4b796993 100644 --- a/src/index_controller/local_index_controller/update_store.rs +++ b/src/index_controller/local_index_controller/update_store.rs @@ -192,16 +192,6 @@ where } } - /// The id and metadata of the update that is currently being processed, - /// `None` if no update is being processed. - pub fn processing_update(&self) -> heed::Result>> { - let rtxn = self.env.read_txn()?; - match self.pending_meta.first(&rtxn)? { - Some((_, meta)) => Ok(Some(meta)), - None => Ok(None), - } - } - /// Execute the user defined function with the meta-store iterators, the first /// iterator is the *processed* meta one, the second the *aborted* meta one /// and, the last is the *pending* meta one. diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index f1ba8f7ce..0ea654dfb 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,5 +1,5 @@ mod local_index_controller; -pub mod updates; +mod updates; pub use local_index_controller::LocalIndexController; @@ -12,7 +12,9 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; -use updates::{Processed, Processing, Failed, UpdateStatus}; +pub use updates::{Processed, Processing, Failed}; + +pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -85,9 +87,10 @@ pub enum UpdateResult { } /// The `IndexController` is in charge of the access to the underlying indices. It splits the logic -/// for read access which is provided, and write access which must be provided. This allows the -/// implementer to define the behaviour of write accesses to the indices, and abstract the -/// scheduling of the updates. The implementer must be able to provide an instance of `IndexStore` +/// for read access which is provided thanks to an handle to the index, and write access which must +/// be provided. This allows the implementer to define the behaviour of write accesses to the +/// indices, and abstract the scheduling of the updates. The implementer must be able to provide an +/// instance of `IndexStore` pub trait IndexController { /* @@ -106,11 +109,11 @@ pub trait IndexController { method: IndexDocumentsMethod, format: UpdateFormat, data: &[u8], - ) -> anyhow::Result>; + ) -> anyhow::Result; /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. - fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result>; + fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; /// Create an index with the given `index_uid`. fn create_index>(&self, index_uid: S) -> Result<()>; @@ -133,9 +136,9 @@ pub trait IndexController { todo!() } - /// Returns, if it exists, an `IndexView` to the requested index. - fn index(&self, uid: impl AsRef) -> anyhow::Result>>; + /// Returns, if it exists, the `Index` with the povided name. + fn index(&self, name: impl AsRef) -> anyhow::Result>>; - fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>>; - fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>>; + fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>; + fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>; } diff --git a/src/lib.rs b/src/lib.rs index df9381914..d542fd6d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,6 @@ pub mod error; pub mod helpers; pub mod option; pub mod routes; -//mod updates; mod index_controller; use actix_http::Error; From f8f02af23e372ae0b07a44ba1eaf8503e62f0f67 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Feb 2021 13:21:15 +0100 Subject: [PATCH 030/522] incorporate review changes --- Cargo.lock | 98 ------------------- Cargo.toml | 5 +- src/data/search.rs | 2 +- src/data/updates.rs | 15 ++- .../local_index_controller/index_store.rs | 40 ++++---- src/index_controller/mod.rs | 1 - src/option.rs | 16 +-- 7 files changed, 43 insertions(+), 134 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f036664f4..0b8128118 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -808,12 +808,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" -[[package]] -name = "difference" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" - [[package]] name = "digest" version = "0.8.1" @@ -838,12 +832,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" -[[package]] -name = "downcast" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" - [[package]] name = "either" version = "1.6.1" @@ -949,15 +937,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" -dependencies = [ - "num-traits", -] - [[package]] name = "fnv" version = "1.0.7" @@ -974,12 +953,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fragile" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" - [[package]] name = "fs_extra" version = "1.2.0" @@ -1679,10 +1652,7 @@ dependencies = [ "memmap", "milli", "mime", - "mockall", - "obkv", "once_cell", - "page_size", "rand 0.7.3", "rayon", "regex", @@ -1880,33 +1850,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "mockall" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619634fd9149c4a06e66d8fd9256e85326d8eeee75abee4565ff76c92e4edfe0" -dependencies = [ - "cfg-if 1.0.0", - "downcast", - "fragile", - "lazy_static", - "mockall_derive", - "predicates", - "predicates-tree", -] - -[[package]] -name = "mockall_derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83714c95dbf4c24202f0f1b208f0f248e6bd65abfa8989303611a71c0f781548" -dependencies = [ - "cfg-if 1.0.0", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "near-proximity" version = "0.1.0" @@ -1938,12 +1881,6 @@ dependencies = [ "libc", ] -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "num-integer" version = "0.1.44" @@ -2212,35 +2149,6 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" -[[package]] -name = "predicates" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" -dependencies = [ - "difference", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" - -[[package]] -name = "predicates-tree" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" -dependencies = [ - "predicates-core", - "treeline", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3218,12 +3126,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "treeline" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" - [[package]] name = "trust-dns-proto" version = "0.19.6" diff --git a/Cargo.toml b/Cargo.toml index ad2d034ad..4668a6897 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = { version = "0.10.6", default-features = false, features = ["serde", "lmdb", "sync-read-txn"] } +heed = "0.10.6" http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" @@ -55,8 +55,6 @@ tar = "0.4.29" tempfile = "3.1.0" tokio = { version = "0.2", features = ["full"] } dashmap = "4.0.2" -page_size = "0.4.2" -obkv = "0.1.1" uuid = "0.8.2" itertools = "0.10.0" @@ -72,7 +70,6 @@ serde_url_params = "0.2.0" tempdir = "0.3.7" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } tokio = { version = "0.2", features = ["macros", "time"] } -mockall = "0.9.0" [features] default = ["sentry"] diff --git a/src/data/search.rs b/src/data/search.rs index 2e05988aa..d0858d704 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -51,7 +51,7 @@ impl SearchQuery { if let Some(ref condition) = self.facet_condition { if !condition.trim().is_empty() { - let condition = FacetCondition::from_str(&rtxn, &index, &condition).unwrap(); + let condition = FacetCondition::from_str(&rtxn, &index, &condition)?; search.facet_condition(condition); } } diff --git a/src/data/updates.rs b/src/data/updates.rs index 27fc6537e..06aed8faa 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -25,7 +25,9 @@ impl Data { let file = tokio::fs::File::from_std(file?); let mut encoder = GzipEncoder::new(file); + let mut empty_update = true; while let Some(result) = stream.next().await { + empty_update = false; let bytes = &*result?; encoder.write_all(&bytes[..]).await?; } @@ -34,10 +36,19 @@ impl Data { let mut file = encoder.into_inner(); file.sync_all().await?; let file = file.into_std().await; - let mmap = unsafe { memmap::Mmap::map(&file)? }; + let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.add_documents(index, method, format, &mmap[..])).await??; + let update = tokio::task::spawn_blocking(move ||{ + let mmap; + let bytes = if empty_update { + &[][..] + } else { + mmap = unsafe { memmap::Mmap::map(&file)? }; + &mmap + }; + index_controller.add_documents(index, method, format, &bytes) + }).await??; Ok(update.into()) } diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 16df83d2c..483b6f5d6 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -17,8 +17,8 @@ type UpdateStore = super::update_store::UpdateStore, + name_to_uuid_meta: Database, uuid_to_index: DashMap, Arc)>, uuid_to_index_db: Database>, @@ -76,7 +76,7 @@ impl IndexStore { Ok(Self { env, - name_to_uuid_db: name_to_uid_db, + name_to_uuid_meta: name_to_uid_db, uuid_to_index: uid_to_index, uuid_to_index_db: uid_to_index_db, @@ -86,7 +86,7 @@ impl IndexStore { } fn index_uuid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uuid_db.get(txn, name.as_ref())? { + match self.name_to_uuid_meta.get(txn, name.as_ref())? { Some(bytes) => { let uuid = Uuid::from_slice(bytes)?; Ok(Some(uuid)) @@ -115,7 +115,7 @@ impl IndexStore { } } - fn _get_index(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { + fn get_index_txn(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { match self.index_uuid(&txn, name)? { Some(uid) => self.retrieve_index(&txn, uid), None => Ok(None), @@ -124,7 +124,7 @@ impl IndexStore { pub fn index(&self, name: impl AsRef) -> anyhow::Result, Arc)>> { let txn = self.env.read_txn()?; - self._get_index(&txn, name) + self.get_index_txn(&txn, name) } pub fn get_or_create_index( @@ -134,7 +134,7 @@ impl IndexStore { index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { let mut txn = self.env.write_txn()?; - match self._get_index(&txn, name.as_ref())? { + match self.get_index_txn(&txn, name.as_ref())? { Some(res) => Ok(res), None => { let uuid = Uuid::new_v4(); @@ -168,9 +168,9 @@ impl IndexStore { update_size: u64, index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { - let meta = IndexMeta { update_size, index_size, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: update_size, index_store_size: index_size, uuid: uuid.clone() }; - self.name_to_uuid_db.put(txn, name.as_ref(), uuid.as_bytes())?; + self.name_to_uuid_meta.put(txn, name.as_ref(), uuid.as_bytes())?; self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?; let path = self.env.path(); @@ -247,7 +247,7 @@ mod test { // insert an uuid in the the name_to_uuid_db: let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.name_to_uuid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap(); txn.commit().unwrap(); // check that the uuid is there @@ -264,7 +264,7 @@ mod test { let txn = store.env.read_txn().unwrap(); assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); - let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; let mut txn = store.env.write_txn().unwrap(); store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); @@ -286,9 +286,9 @@ mod test { assert!(store.index(&name).unwrap().is_none()); let uuid = Uuid::new_v4(); - let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; let mut txn = store.env.write_txn().unwrap(); - store.name_to_uuid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap(); store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); @@ -303,11 +303,11 @@ mod test { store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap(); let txn = store.env.read_txn().unwrap(); - let uuid = store.name_to_uuid_db.get(&txn, &name).unwrap(); + let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); } @@ -322,11 +322,11 @@ mod test { let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); store.create_index(&mut txn, uuid, name, update_size, index_size).unwrap(); - let uuid = store.name_to_uuid_db.get(&txn, &name).unwrap(); + let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = IndexMeta { update_size , index_size, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: update_size , index_store_size: index_size, uuid: uuid.clone() }; assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0ea654dfb..d348ee876 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -16,7 +16,6 @@ pub use updates::{Processed, Processing, Failed}; pub type UpdateStatus = updates::UpdateStatus; - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] pub enum UpdateMeta { diff --git a/src/option.rs b/src/option.rs index dffa7d483..38f99880e 100644 --- a/src/option.rs +++ b/src/option.rs @@ -19,11 +19,11 @@ pub struct IndexerOpts { #[structopt(long, default_value = "100000")] // 100k pub log_every_n: usize, - /// MTBL max number of chunks in bytes. + /// Grenad max number of chunks in bytes. #[structopt(long)] pub max_nb_chunks: Option, - /// The maximum amount of memory to use for the MTBL buffer. It is recommended + /// The maximum amount of memory to use for the Grenad buffer. It is recommended /// to use something like 80%-90% of the available memory. /// /// It is automatically split by the number of jobs e.g. if you use 7 jobs @@ -37,7 +37,7 @@ pub struct IndexerOpts { pub linked_hash_map_size: usize, /// The name of the compression algorithm to use when compressing intermediate - /// chunks during indexing documents. + /// Grenad chunks while indexing documents. /// /// Choosing a fast algorithm will make the indexing faster but may consume more memory. #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] @@ -55,7 +55,7 @@ pub struct IndexerOpts { #[structopt(long, default_value = "4 GiB")] pub chunk_fusing_shrink_size: Byte, - /// Enable the chunk fusing or not, this reduces the amount of disk used by a factor of 2. + /// Enable the chunk fusing or not, this reduces the amount of disk space used. #[structopt(long)] pub enable_chunk_fusing: bool, @@ -68,13 +68,13 @@ pub struct IndexerOpts { impl Default for IndexerOpts { fn default() -> Self { Self { - log_every_n: 0, + log_every_n: 100_000, max_nb_chunks: None, - max_memory: Byte::from_str("0Kb").unwrap(), - linked_hash_map_size: 0, + max_memory: Byte::from_str("1GiB").unwrap(), + linked_hash_map_size: 500, chunk_compression_type: CompressionType::None, chunk_compression_level: None, - chunk_fusing_shrink_size: Byte::from_str("0Kb").unwrap(), + chunk_fusing_shrink_size: Byte::from_str("4GiB").unwrap(), enable_chunk_fusing: false, indexing_jobs: None, } From d43dc4824cf9aab5e122d1915944985d5ab8d871 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Feb 2021 17:44:20 +0100 Subject: [PATCH 031/522] implement list indexes --- src/data/mod.rs | 8 ++- .../local_index_controller/index_store.rs | 67 +++++++++++++++---- .../local_index_controller/mod.rs | 27 +++++++- src/index_controller/mod.rs | 14 ++++ src/index_controller/updates.rs | 7 ++ src/routes/index.rs | 23 ++++--- 6 files changed, 120 insertions(+), 26 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index de24d0a06..cada4f559 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -9,8 +9,8 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexController, LocalIndexController}; -use crate::{option::Opt, index_controller::Settings}; +use crate::index_controller::{IndexController, LocalIndexController, IndexMetadata, Settings}; +use crate::option::Opt; #[derive(Clone)] pub struct Data { @@ -114,6 +114,10 @@ impl Data { }) } + pub fn list_indexes(&self) -> anyhow::Result> { + self.index_controller.list_indexes() + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 483b6f5d6..138fe57fc 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -2,8 +2,10 @@ use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use chrono::{DateTime, Utc}; use dashmap::{DashMap, mapref::entry::Entry}; use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; +use log::error; use milli::Index; use rayon::ThreadPool; use serde::{Serialize, Deserialize}; @@ -16,10 +18,11 @@ use super::{UpdateMeta, UpdateResult}; type UpdateStore = super::update_store::UpdateStore; #[derive(Serialize, Deserialize, Debug, PartialEq)] -struct IndexMeta { - update_store_size: u64, - index_store_size: u64, - uuid: Uuid, +pub struct IndexMeta { + update_size: u64, + index_size: u64, + pub uuid: Uuid, + pub created_at: DateTime, } impl IndexMeta { @@ -168,7 +171,8 @@ impl IndexStore { update_size: u64, index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { - let meta = IndexMeta { update_store_size: update_size, index_store_size: index_size, uuid: uuid.clone() }; + let created_at = Utc::now(); + let meta = IndexMeta { update_size, index_size, uuid: uuid.clone(), created_at }; self.name_to_uuid_meta.put(txn, name.as_ref(), uuid.as_bytes())?; self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?; @@ -186,6 +190,29 @@ impl IndexStore { Ok((index, update_store)) } + + /// Returns each index associated with it's metadata; + pub fn list_indexes(&self) -> anyhow::Result> { + let txn = self.env.read_txn()?; + let indexes = self.name_to_uuid_db + .iter(&txn)? + .filter_map(|entry| entry + .map_err(|e| { + error!("error decoding entry while listing indexes: {}", e); + e + }) + .ok()) + .map(|(name, uuid)| { + let meta = self.uuid_to_index_db + .get(&txn, &uuid) + .ok() + .flatten() + .unwrap_or_else(|| panic!("corrupted database, index {} should exist.", name)); + (name.to_owned(), meta) + }) + .collect(); + Ok(indexes) + } } fn open_or_create_database(env: &Env, name: Option<&str>) -> anyhow::Result> { @@ -264,7 +291,12 @@ mod test { let txn = store.env.read_txn().unwrap(); assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); - let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { + update_size: 4096 * 100, + index_size: 4096 * 100, + uuid: uuid.clone(), + created_at: Utc::now(), + }; let mut txn = store.env.write_txn().unwrap(); store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); @@ -286,7 +318,12 @@ mod test { assert!(store.index(&name).unwrap().is_none()); let uuid = Uuid::new_v4(); - let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { + update_size: 4096 * 100, + index_size: 4096 * 100, + uuid: uuid.clone(), + created_at: Utc::now(), + }; let mut txn = store.env.write_txn().unwrap(); store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap(); store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); @@ -301,14 +338,18 @@ mod test { let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); let name = "foobar"; - store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap(); + let update_size = 4096 * 100; + let index_size = 4096 * 100; + store.get_or_create_index(&name, update_size, index_size).unwrap(); let txn = store.env.read_txn().unwrap(); let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; - assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap(); + assert_eq!(meta.update_size, update_size); + assert_eq!(meta.index_size, index_size); + assert_eq!(meta.uuid, uuid); } #[test] @@ -326,8 +367,10 @@ mod test { assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = IndexMeta { update_store_size: update_size , index_store_size: index_size, uuid: uuid.clone() }; - assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap(); + assert_eq!(meta.update_size, update_size); + assert_eq!(meta.index_size, index_size); + assert_eq!(meta.uuid, uuid); } } } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index b59eb2a99..6d6700639 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -13,7 +13,7 @@ use crate::option::IndexerOpts; use index_store::IndexStore; use super::IndexController; use super::updates::UpdateStatus; -use super::{UpdateMeta, UpdateResult}; +use super::{UpdateMeta, UpdateResult, IndexMetadata}; pub struct LocalIndexController { indexes: IndexStore, @@ -102,4 +102,29 @@ impl IndexController for LocalIndexController { } } + + fn list_indexes(&self) -> anyhow::Result> { + let metas = self.indexes.list_indexes()?; + let mut output_meta = Vec::new(); + for (name, meta) in metas { + let created_at = meta.created_at; + let uuid = meta.uuid; + let updated_at = self + .all_update_status(&name)? + .iter() + .filter_map(|u| u.processed().map(|u| u.processed_at)) + .max() + .unwrap_or(created_at); + + let index_meta = IndexMetadata { + name, + created_at, + updated_at, + uuid, + primary_key: None, + }; + output_meta.push(index_meta); + } + Ok(output_meta) + } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index d348ee876..38ff149bf 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -8,14 +8,26 @@ use std::num::NonZeroUsize; use std::sync::Arc; use anyhow::Result; +use chrono::{DateTime, Utc}; use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; +use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; pub type UpdateStatus = updates::UpdateStatus; +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IndexMetadata { + pub name: String, + uuid: Uuid, + created_at: DateTime, + updated_at: DateTime, + pub primary_key: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] pub enum UpdateMeta { @@ -140,4 +152,6 @@ pub trait IndexController { fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>; fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>; + + fn list_indexes(&self) -> anyhow::Result>; } diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs index 4eb94dc4a..900987ba6 100644 --- a/src/index_controller/updates.rs +++ b/src/index_controller/updates.rs @@ -134,6 +134,13 @@ impl UpdateStatus { UpdateStatus::Failed(u) => u.id(), } } + + pub fn processed(&self) -> Option<&Processed> { + match self { + UpdateStatus::Processed(p) => Some(p), + _ => None, + } + } } impl From> for UpdateStatus { diff --git a/src/routes/index.rs b/src/routes/index.rs index 774039f2b..653d51475 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -19,19 +19,20 @@ pub fn services(cfg: &mut web::ServiceConfig) { .service(get_all_updates_status); } -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct IndexResponse { - pub name: String, - pub uid: String, - created_at: DateTime, - updated_at: DateTime, - pub primary_key: Option, -} #[get("/indexes", wrap = "Authentication::Private")] -async fn list_indexes(_data: web::Data) -> Result { - todo!() +async fn list_indexes(data: web::Data) -> Result { + match data.list_indexes() { + Ok(indexes) => { + let json = serde_json::to_string(&indexes).unwrap(); + Ok(HttpResponse::Ok().body(&json)) + } + Err(e) => { + error!("error listing indexes: {}", e); + unimplemented!() + } + } + } #[get("/indexes/{index_uid}", wrap = "Authentication::Private")] From f98830669186c0aaba33eb499486318ddc4511e1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Feb 2021 20:12:48 +0100 Subject: [PATCH 032/522] implement create index --- .../local_index_controller/index_store.rs | 33 ++++++++++++++++--- .../local_index_controller/mod.rs | 5 +-- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 138fe57fc..7303a7849 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -2,6 +2,7 @@ use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use anyhow::bail; use chrono::{DateTime, Utc}; use dashmap::{DashMap, mapref::entry::Entry}; use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; @@ -141,7 +142,7 @@ impl IndexStore { Some(res) => Ok(res), None => { let uuid = Uuid::new_v4(); - let result = self.create_index(&mut txn, uuid, name, update_size, index_size)?; + let result = self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; // If we fail to commit the transaction, we must delete the database from the // file-system. if let Err(e) = txn.commit() { @@ -164,7 +165,7 @@ impl IndexStore { self.uuid_to_index.remove(&uuid); } - fn create_index( &self, + fn create_index_txn( &self, txn: &mut RwTxn, uuid: Uuid, name: impl AsRef, @@ -191,6 +192,30 @@ impl IndexStore { Ok((index, update_store)) } + /// Same a get or create, but returns an error if the index already exists. + pub fn create_index( + &self, + name: impl AsRef, + update_size: u64, + index_size: u64, + ) -> anyhow::Result<(Arc, Arc)> { + let uuid = Uuid::new_v4(); + let mut txn = self.env.write_txn()?; + + if self.name_to_uuid_db.get(&txn, name.as_ref())?.is_some() { + bail!("cannot create index {:?}: an index with this name already exists.") + } + + let result = self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; + // If we fail to commit the transaction, we must delete the database from the + // file-system. + if let Err(e) = txn.commit() { + self.clean_db(uuid); + return Err(e)?; + } + Ok(result) + } + /// Returns each index associated with it's metadata; pub fn list_indexes(&self) -> anyhow::Result> { let txn = self.env.read_txn()?; @@ -362,8 +387,8 @@ mod test { let index_size = 4096 * 100; let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.create_index(&mut txn, uuid, name, update_size, index_size).unwrap(); - let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap(); + store.create_index_txn(&mut txn, uuid, name, update_size, index_size).unwrap(); + let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 6d6700639..ec70c4443 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -58,8 +58,9 @@ impl IndexController for LocalIndexController { Ok(pending.into()) } - fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { - todo!() + fn create_index>(&self, index_uid: S) -> anyhow::Result<()> { + self.indexes.create_index(index_uid, self.update_db_size, self.index_db_size)?; + Ok(()) } fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { From 8d462afb79b3048b722367365ea7f0cbeb81d831 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Feb 2021 20:20:51 +0100 Subject: [PATCH 033/522] add tests for list index and create index. --- .../local_index_controller/mod.rs | 14 ++++++++++ src/index_controller/mod.rs | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index ec70c4443..21139e636 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -129,3 +129,17 @@ impl IndexController for LocalIndexController { Ok(output_meta) } } + +#[cfg(test)] +mod test { + use super::*; + use tempfile::tempdir; + use crate::make_index_controller_tests; + + make_index_controller_tests!({ + let options = IndexerOpts::default(); + let path = tempdir().unwrap(); + let size = 4096 * 100; + LocalIndexController::new(path, options, size, size).unwrap() + }); +} diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 38ff149bf..720c52cdc 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -155,3 +155,30 @@ pub trait IndexController { fn list_indexes(&self) -> anyhow::Result>; } + + +#[cfg(test)] +#[macro_use] +pub(crate) mod test { + use super::*; + + #[macro_export] + macro_rules! make_index_controller_tests { + ($controller_buider:block) => { + #[test] + fn test_create_and_list_indexes() { + crate::index_controller::test::create_and_list_indexes($controller_buider); + } + }; + } + + pub(crate) fn create_and_list_indexes(controller: S) { + controller.create_index("test_index").unwrap(); + controller.create_index("test_index2").unwrap(); + + let indexes = controller.list_indexes().unwrap(); + assert_eq!(indexes.len(), 2); + assert_eq!(indexes[0].name, "test_index"); + assert_eq!(indexes[1].name, "test_index2"); + } +} From f1c09a54be15648ade94fc9354199b546c91f514 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Feb 2021 12:34:12 +0100 Subject: [PATCH 034/522] implement get index meta --- src/data/mod.rs | 7 +++++++ src/error.rs | 7 +++++++ src/routes/document.rs | 10 +--------- src/routes/index.rs | 14 +++++++++++--- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index cada4f559..61945ee2a 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -118,6 +118,13 @@ impl Data { self.index_controller.list_indexes() } + pub fn index(&self, name: impl AsRef) -> anyhow::Result> { + Ok(self + .list_indexes()? + .into_iter() + .find(|i| i.name == name.as_ref())) + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/src/error.rs b/src/error.rs index c3533bcef..33aa06d3c 100644 --- a/src/error.rs +++ b/src/error.rs @@ -28,6 +28,13 @@ impl fmt::Display for ResponseError { } } +// TODO: remove this when implementing actual error handling +impl From for ResponseError { + fn from(other: anyhow::Error) -> ResponseError { + ResponseError { inner: Box::new(Error::NotFound(other.to_string())) } + } +} + impl From for ResponseError { fn from(error: Error) -> ResponseError { ResponseError { inner: Box::new(error) } diff --git a/src/routes/document.rs b/src/routes/document.rs index aeec0e5df..dcc669f85 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -83,15 +83,6 @@ async fn get_all_documents( todo!() } -//fn find_primary_key(document: &IndexMap) -> Option { - //for key in document.keys() { - //if key.to_lowercase().contains("id") { - //return Some(key.to_string()); - //} - //} - //None -//} - #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateDocumentsQuery { @@ -150,6 +141,7 @@ async fn add_documents_default( _params: web::Query, _body: web::Json>, ) -> Result { + error!("Unknown document type"); todo!() } diff --git a/src/routes/index.rs b/src/routes/index.rs index 653d51475..5e2574267 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -37,10 +37,18 @@ async fn list_indexes(data: web::Data) -> Result, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() + match data.index(&path.index_uid)? { + Some(meta) => { + let json = serde_json::to_string(&meta).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + None => { + unimplemented!() + } + } } #[derive(Debug, Deserialize)] From f18e795124b5cb5af7486b366eb27d88ca6c7313 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Feb 2021 15:09:43 +0100 Subject: [PATCH 035/522] fix rebase --- .../local_index_controller/index_store.rs | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 7303a7849..eca18444f 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -20,8 +20,8 @@ type UpdateStore = super::update_store::UpdateStore, } @@ -54,9 +54,9 @@ impl IndexMeta { pub struct IndexStore { env: Env, - name_to_uuid_meta: Database, + name_to_uuid: Database, uuid_to_index: DashMap, Arc)>, - uuid_to_index_db: Database>, + uuid_to_index_meta: Database>, thread_pool: Arc, indexer_options: IndexerOpts, @@ -69,9 +69,9 @@ impl IndexStore { .max_dbs(2) .open(path)?; - let uid_to_index = DashMap::new(); - let name_to_uid_db = open_or_create_database(&env, Some("name_to_uid"))?; - let uid_to_index_db = open_or_create_database(&env, Some("uid_to_index_db"))?; + let uuid_to_index = DashMap::new(); + let name_to_uuid = open_or_create_database(&env, Some("name_to_uid"))?; + let uuid_to_index_meta = open_or_create_database(&env, Some("uid_to_index_db"))?; let thread_pool = rayon::ThreadPoolBuilder::new() .num_threads(indexer_options.indexing_jobs.unwrap_or(0)) @@ -80,9 +80,9 @@ impl IndexStore { Ok(Self { env, - name_to_uuid_meta: name_to_uid_db, - uuid_to_index: uid_to_index, - uuid_to_index_db: uid_to_index_db, + name_to_uuid, + uuid_to_index, + uuid_to_index_meta, thread_pool, indexer_options, @@ -90,7 +90,7 @@ impl IndexStore { } fn index_uuid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uuid_meta.get(txn, name.as_ref())? { + match self.name_to_uuid.get(txn, name.as_ref())? { Some(bytes) => { let uuid = Uuid::from_slice(bytes)?; Ok(Some(uuid)) @@ -102,7 +102,7 @@ impl IndexStore { fn retrieve_index(&self, txn: &RoTxn, uid: Uuid) -> anyhow::Result, Arc)>> { match self.uuid_to_index.entry(uid.clone()) { Entry::Vacant(entry) => { - match self.uuid_to_index_db.get(txn, uid.as_bytes())? { + match self.uuid_to_index_meta.get(txn, uid.as_bytes())? { Some(meta) => { let path = self.env.path(); let (index, updates) = meta.open(path, self.thread_pool.clone(), &self.indexer_options)?; @@ -169,14 +169,14 @@ impl IndexStore { txn: &mut RwTxn, uuid: Uuid, name: impl AsRef, - update_size: u64, - index_size: u64, + update_store_size: u64, + index_store_size: u64, ) -> anyhow::Result<(Arc, Arc)> { let created_at = Utc::now(); - let meta = IndexMeta { update_size, index_size, uuid: uuid.clone(), created_at }; + let meta = IndexMeta { update_store_size, index_store_size, uuid: uuid.clone(), created_at }; - self.name_to_uuid_meta.put(txn, name.as_ref(), uuid.as_bytes())?; - self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?; + self.name_to_uuid.put(txn, name.as_ref(), uuid.as_bytes())?; + self.uuid_to_index_meta.put(txn, uuid.as_bytes(), &meta)?; let path = self.env.path(); let (index, update_store) = match meta.open(path, self.thread_pool.clone(), &self.indexer_options) { @@ -202,7 +202,7 @@ impl IndexStore { let uuid = Uuid::new_v4(); let mut txn = self.env.write_txn()?; - if self.name_to_uuid_db.get(&txn, name.as_ref())?.is_some() { + if self.name_to_uuid.get(&txn, name.as_ref())?.is_some() { bail!("cannot create index {:?}: an index with this name already exists.") } @@ -219,7 +219,7 @@ impl IndexStore { /// Returns each index associated with it's metadata; pub fn list_indexes(&self) -> anyhow::Result> { let txn = self.env.read_txn()?; - let indexes = self.name_to_uuid_db + let indexes = self.name_to_uuid .iter(&txn)? .filter_map(|entry| entry .map_err(|e| { @@ -228,7 +228,7 @@ impl IndexStore { }) .ok()) .map(|(name, uuid)| { - let meta = self.uuid_to_index_db + let meta = self.uuid_to_index_meta .get(&txn, &uuid) .ok() .flatten() @@ -299,7 +299,7 @@ mod test { // insert an uuid in the the name_to_uuid_db: let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.name_to_uuid.put(&mut txn, &name, uuid.as_bytes()).unwrap(); txn.commit().unwrap(); // check that the uuid is there @@ -317,13 +317,13 @@ mod test { assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); let meta = IndexMeta { - update_size: 4096 * 100, - index_size: 4096 * 100, + update_store_size: 4096 * 100, + index_store_size: 4096 * 100, uuid: uuid.clone(), created_at: Utc::now(), }; let mut txn = store.env.write_txn().unwrap(); - store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + store.uuid_to_index_meta.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); // the index cache should be empty @@ -344,14 +344,14 @@ mod test { let uuid = Uuid::new_v4(); let meta = IndexMeta { - update_size: 4096 * 100, - index_size: 4096 * 100, + update_store_size: 4096 * 100, + index_store_size: 4096 * 100, uuid: uuid.clone(), created_at: Utc::now(), }; let mut txn = store.env.write_txn().unwrap(); - store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap(); - store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + store.name_to_uuid.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.uuid_to_index_meta.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); assert!(store.index(&name).unwrap().is_some()); @@ -363,17 +363,17 @@ mod test { let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); let name = "foobar"; - let update_size = 4096 * 100; - let index_size = 4096 * 100; - store.get_or_create_index(&name, update_size, index_size).unwrap(); + let update_store_size = 4096 * 100; + let index_store_size = 4096 * 100; + store.get_or_create_index(&name, update_store_size, index_store_size).unwrap(); let txn = store.env.read_txn().unwrap(); - let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap(); + let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap(); - assert_eq!(meta.update_size, update_size); - assert_eq!(meta.index_size, index_size); + let meta = store.uuid_to_index_meta.get(&txn, uuid.as_bytes()).unwrap().unwrap(); + assert_eq!(meta.update_store_size, update_store_size); + assert_eq!(meta.index_store_size, index_store_size); assert_eq!(meta.uuid, uuid); } @@ -383,18 +383,18 @@ mod test { let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); let name = "foobar"; - let update_size = 4096 * 100; - let index_size = 4096 * 100; + let update_store_size = 4096 * 100; + let index_store_size = 4096 * 100; let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.create_index_txn(&mut txn, uuid, name, update_size, index_size).unwrap(); + store.create_index_txn(&mut txn, uuid, name, update_store_size, index_store_size).unwrap(); let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap(); - assert_eq!(meta.update_size, update_size); - assert_eq!(meta.index_size, index_size); + let meta = store.uuid_to_index_meta.get(&txn, uuid.as_bytes()).unwrap().unwrap(); + assert_eq!(meta.update_store_size, update_store_size); + assert_eq!(meta.index_store_size, index_store_size); assert_eq!(meta.uuid, uuid); } } From ed44e684ccc70062ff8514db98710f1f43a540b8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Feb 2021 15:28:52 +0100 Subject: [PATCH 036/522] review fixes --- .../local_index_controller/index_store.rs | 26 +++++++------------ src/index_controller/mod.rs | 2 +- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index eca18444f..937a0cac0 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -192,7 +192,7 @@ impl IndexStore { Ok((index, update_store)) } - /// Same a get or create, but returns an error if the index already exists. + /// Same as `get_or_create`, but returns an error if the index already exists. pub fn create_index( &self, name: impl AsRef, @@ -219,23 +219,17 @@ impl IndexStore { /// Returns each index associated with it's metadata; pub fn list_indexes(&self) -> anyhow::Result> { let txn = self.env.read_txn()?; - let indexes = self.name_to_uuid + let metas = self.name_to_uuid .iter(&txn)? .filter_map(|entry| entry - .map_err(|e| { - error!("error decoding entry while listing indexes: {}", e); - e - }) - .ok()) - .map(|(name, uuid)| { - let meta = self.uuid_to_index_meta - .get(&txn, &uuid) - .ok() - .flatten() - .unwrap_or_else(|| panic!("corrupted database, index {} should exist.", name)); - (name.to_owned(), meta) - }) - .collect(); + .map_err(|e| { error!("error decoding entry while listing indexes: {}", e); e }).ok()); + let mut indexes = Vec::new(); + for (name, uuid) in metas { + let meta = self.uuid_to_index_meta + .get(&txn, &uuid)? + .unwrap_or_else(|| panic!("corrupted database, index {} should exist.", name)); + indexes.push((name.to_owned(), meta)); + } Ok(indexes) } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 720c52cdc..13ecc1486 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -25,7 +25,7 @@ pub struct IndexMetadata { uuid: Uuid, created_at: DateTime, updated_at: DateTime, - pub primary_key: Option, + primary_key: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] From ec047eefd2abec39c4d4c3c17c8da4c87c1b917f Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Feb 2021 10:47:34 +0100 Subject: [PATCH 037/522] implement create index --- src/data/mod.rs | 5 +++ .../local_index_controller/index_store.rs | 28 +++++++++----- .../local_index_controller/mod.rs | 38 ++++++++++++++++--- src/index_controller/mod.rs | 2 +- src/routes/index.rs | 18 ++++++--- 5 files changed, 69 insertions(+), 22 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 61945ee2a..65364df00 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -125,6 +125,11 @@ impl Data { .find(|i| i.name == name.as_ref())) } + pub fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { + let meta = self.index_controller.create_index(name, primary_key)?; + Ok(meta) + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 937a0cac0..d28228efd 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -2,7 +2,7 @@ use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use anyhow::bail; +use anyhow::{bail, Context}; use chrono::{DateTime, Utc}; use dashmap::{DashMap, mapref::entry::Entry}; use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; @@ -142,14 +142,14 @@ impl IndexStore { Some(res) => Ok(res), None => { let uuid = Uuid::new_v4(); - let result = self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; + let (index, updates, _) = self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; // If we fail to commit the transaction, we must delete the database from the // file-system. if let Err(e) = txn.commit() { self.clean_db(uuid); return Err(e)?; } - Ok(result) + Ok((index, updates)) }, } } @@ -171,7 +171,7 @@ impl IndexStore { name: impl AsRef, update_store_size: u64, index_store_size: u64, - ) -> anyhow::Result<(Arc, Arc)> { + ) -> anyhow::Result<(Arc, Arc, IndexMeta)> { let created_at = Utc::now(); let meta = IndexMeta { update_store_size, index_store_size, uuid: uuid.clone(), created_at }; @@ -189,7 +189,7 @@ impl IndexStore { self.uuid_to_index.insert(uuid, (index.clone(), update_store.clone())); - Ok((index, update_store)) + Ok((index, update_store, meta)) } /// Same as `get_or_create`, but returns an error if the index already exists. @@ -198,7 +198,7 @@ impl IndexStore { name: impl AsRef, update_size: u64, index_size: u64, - ) -> anyhow::Result<(Arc, Arc)> { + ) -> anyhow::Result<(Arc, Arc, IndexMeta)> { let uuid = Uuid::new_v4(); let mut txn = self.env.write_txn()?; @@ -216,8 +216,10 @@ impl IndexStore { Ok(result) } - /// Returns each index associated with it's metadata; - pub fn list_indexes(&self) -> anyhow::Result> { + /// Returns each index associated with its metadata: + /// (index_name, IndexMeta, primary_key) + /// This method will force all the indexes to be loaded. + pub fn list_indexes(&self) -> anyhow::Result)>> { let txn = self.env.read_txn()?; let metas = self.name_to_uuid .iter(&txn)? @@ -225,10 +227,16 @@ impl IndexStore { .map_err(|e| { error!("error decoding entry while listing indexes: {}", e); e }).ok()); let mut indexes = Vec::new(); for (name, uuid) in metas { + // get index to retrieve primary key + let (index, _) = self.get_index_txn(&txn, name)? + .with_context(|| format!("could not load index {:?}", name))?; + let primary_key = index.primary_key(&index.read_txn()?)? + .map(String::from); + // retieve meta let meta = self.uuid_to_index_meta .get(&txn, &uuid)? - .unwrap_or_else(|| panic!("corrupted database, index {} should exist.", name)); - indexes.push((name.to_owned(), meta)); + .with_context(|| format!("could not retieve meta for index {:?}", name))?; + indexes.push((name.to_owned(), meta, primary_key)); } Ok(indexes) } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 21139e636..9624a9c64 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -5,7 +5,7 @@ mod update_handler; use std::path::Path; use std::sync::Arc; -use anyhow::bail; +use anyhow::{bail, Context}; use itertools::Itertools; use milli::Index; @@ -58,9 +58,24 @@ impl IndexController for LocalIndexController { Ok(pending.into()) } - fn create_index>(&self, index_uid: S) -> anyhow::Result<()> { - self.indexes.create_index(index_uid, self.update_db_size, self.index_db_size)?; - Ok(()) + fn create_index(&self, index_name: impl AsRef, primary_key: Option>) -> anyhow::Result { + let (index, _, meta) = self.indexes.create_index(&index_name, self.update_db_size, self.index_db_size)?; + if let Some(ref primary_key) = primary_key { + if let Err(e) = update_primary_key(index, primary_key).context("error creating index") { + // TODO: creating index could not be completed, delete everything. + Err(e)? + } + } + + let meta = IndexMetadata { + name: index_name.as_ref().to_owned(), + uuid: meta.uuid.clone(), + created_at: meta.created_at, + updated_at: meta.created_at, + primary_key: primary_key.map(|n| n.as_ref().to_owned()), + }; + + Ok(meta) } fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { @@ -107,7 +122,7 @@ impl IndexController for LocalIndexController { fn list_indexes(&self) -> anyhow::Result> { let metas = self.indexes.list_indexes()?; let mut output_meta = Vec::new(); - for (name, meta) in metas { + for (name, meta, primary_key) in metas { let created_at = meta.created_at; let uuid = meta.uuid; let updated_at = self @@ -122,7 +137,7 @@ impl IndexController for LocalIndexController { created_at, updated_at, uuid, - primary_key: None, + primary_key, }; output_meta.push(index_meta); } @@ -130,6 +145,17 @@ impl IndexController for LocalIndexController { } } +fn update_primary_key(index: impl AsRef, primary_key: impl AsRef) -> anyhow::Result<()> { + let index = index.as_ref(); + let mut txn = index.write_txn()?; + if index.primary_key(&txn)?.is_some() { + bail!("primary key already set.") + } + index.put_primary_key(&mut txn, primary_key.as_ref())?; + txn.commit()?; + Ok(()) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 13ecc1486..d5bf873ee 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -127,7 +127,7 @@ pub trait IndexController { fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; /// Create an index with the given `index_uid`. - fn create_index>(&self, index_uid: S) -> Result<()>; + fn create_index(&self, index_uid: impl AsRef, primary_key: Option>) -> Result; /// Delete index with the given `index_uid`, attempting to close it beforehand. fn delete_index>(&self, index_uid: S) -> Result<()>; diff --git a/src/routes/index.rs b/src/routes/index.rs index 5e2574267..d682376e3 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -54,17 +54,25 @@ async fn get_index( #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct IndexCreateRequest { - name: Option, - uid: Option, + name: String, primary_key: Option, } #[post("/indexes", wrap = "Authentication::Private")] async fn create_index( - _data: web::Data, - _body: web::Json, + data: web::Data, + body: web::Json, ) -> Result { - todo!() + match data.create_index(&body.name, body.primary_key.clone()) { + Ok(meta) => { + let json = serde_json::to_string(&meta).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("error creating index: {}", e); + unimplemented!() + } + } } #[derive(Debug, Deserialize)] From e89b11b1fa75501f6cd75ac3fda4a45ba0af7e48 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 9 Feb 2021 11:41:26 +0100 Subject: [PATCH 038/522] create IndexSetting struct need to stabilize the create index trait interface --- src/data/mod.rs | 9 +++-- .../local_index_controller/mod.rs | 11 +++--- src/index_controller/mod.rs | 35 +++++++++++++++++-- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 65364df00..8c512895c 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexController, LocalIndexController, IndexMetadata, Settings}; +use crate::index_controller::{IndexController, LocalIndexController, IndexMetadata, Settings, IndexSettings}; use crate::option::Opt; #[derive(Clone)] @@ -126,7 +126,12 @@ impl Data { } pub fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { - let meta = self.index_controller.create_index(name, primary_key)?; + let settings = IndexSettings { + name: Some(name.as_ref().to_string()), + primary_key: primary_key.map(|s| s.as_ref().to_string()), + }; + + let meta = self.index_controller.create_index(settings)?; Ok(meta) } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 9624a9c64..60a02573f 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -13,7 +13,7 @@ use crate::option::IndexerOpts; use index_store::IndexStore; use super::IndexController; use super::updates::UpdateStatus; -use super::{UpdateMeta, UpdateResult, IndexMetadata}; +use super::{UpdateMeta, UpdateResult, IndexMetadata, IndexSettings}; pub struct LocalIndexController { indexes: IndexStore, @@ -58,9 +58,10 @@ impl IndexController for LocalIndexController { Ok(pending.into()) } - fn create_index(&self, index_name: impl AsRef, primary_key: Option>) -> anyhow::Result { + fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { + let index_name = index_settings.name.context("Missing name for index")?; let (index, _, meta) = self.indexes.create_index(&index_name, self.update_db_size, self.index_db_size)?; - if let Some(ref primary_key) = primary_key { + if let Some(ref primary_key) = index_settings.primary_key { if let Err(e) = update_primary_key(index, primary_key).context("error creating index") { // TODO: creating index could not be completed, delete everything. Err(e)? @@ -68,11 +69,11 @@ impl IndexController for LocalIndexController { } let meta = IndexMetadata { - name: index_name.as_ref().to_owned(), + name: index_name, uuid: meta.uuid.clone(), created_at: meta.created_at, updated_at: meta.created_at, - primary_key: primary_key.map(|n| n.as_ref().to_owned()), + primary_key: index_settings.primary_key, }; Ok(meta) diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index d5bf873ee..2c8dd1226 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -97,6 +97,11 @@ pub enum UpdateResult { Other, } +pub struct IndexSettings { + pub name: Option, + pub primary_key: Option, +} + /// The `IndexController` is in charge of the access to the underlying indices. It splits the logic /// for read access which is provided thanks to an handle to the index, and write access which must /// be provided. This allows the implementer to define the behaviour of write accesses to the @@ -127,7 +132,7 @@ pub trait IndexController { fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; /// Create an index with the given `index_uid`. - fn create_index(&self, index_uid: impl AsRef, primary_key: Option>) -> Result; + fn create_index(&self, index_settings: IndexSettings) -> Result; /// Delete index with the given `index_uid`, attempting to close it beforehand. fn delete_index>(&self, index_uid: S) -> Result<()>; @@ -169,16 +174,40 @@ pub(crate) mod test { fn test_create_and_list_indexes() { crate::index_controller::test::create_and_list_indexes($controller_buider); } + + #[test] + fn test_create_index_with_no_name_is_error() { + crate::index_controller::test::create_index_with_no_name_is_error($controller_buider); + } }; } pub(crate) fn create_and_list_indexes(controller: S) { - controller.create_index("test_index").unwrap(); - controller.create_index("test_index2").unwrap(); + let settings1 = IndexSettings { + name: Some(String::from("test_index")), + primary_key: None, + }; + + let settings2 = IndexSettings { + name: Some(String::from("test_index2")), + primary_key: Some(String::from("foo")), + }; + + controller.create_index(settings1).unwrap(); + controller.create_index(settings2).unwrap(); let indexes = controller.list_indexes().unwrap(); assert_eq!(indexes.len(), 2); assert_eq!(indexes[0].name, "test_index"); assert_eq!(indexes[1].name, "test_index2"); + assert_eq!(indexes[1].primary_key.clone().unwrap(), "foo"); + } + + pub(crate) fn create_index_with_no_name_is_error(controller: S) { + let settings = IndexSettings { + name: None, + primary_key: None, + }; + assert!(controller.create_index(settings).is_err()); } } From 90b930ed7f9c2e8e7344ab685f83fc6e68de12c4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 9 Feb 2021 11:26:52 +0100 Subject: [PATCH 039/522] implement update index implement update index --- src/data/updates.rs | 16 ++++- .../local_index_controller/index_store.rs | 68 ++++++++++++++++++- .../local_index_controller/mod.rs | 38 +++++++++++ src/index_controller/mod.rs | 10 ++- src/routes/index.rs | 14 ++-- 5 files changed, 135 insertions(+), 11 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 06aed8faa..57e9d1864 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -6,7 +6,7 @@ use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; use super::Data; -use crate::index_controller::{IndexController, Settings}; +use crate::index_controller::{IndexController, Settings, IndexSettings, IndexMetadata}; use crate::index_controller::UpdateStatus; impl Data { @@ -70,4 +70,18 @@ impl Data { pub fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { self.index_controller.all_update_status(index) } + + pub fn update_index( + &self, + name: impl AsRef, + primary_key: Option>, + new_name: Option> + ) -> anyhow::Result { + let settings = IndexSettings { + name: new_name.map(|s| s.as_ref().to_string()), + primary_key: primary_key.map(|s| s.as_ref().to_string()), + }; + + self.index_controller.update_index(name, settings) + } } diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index d28228efd..36c864772 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -24,6 +24,7 @@ pub struct IndexMeta { index_store_size: u64, pub uuid: Uuid, pub created_at: DateTime, + pub updated_at: DateTime, } impl IndexMeta { @@ -131,6 +132,52 @@ impl IndexStore { self.get_index_txn(&txn, name) } + /// Use this function to perform an update on an index. + /// This function also puts a lock on what index in allowed to perform an update. + pub fn update_index(&self, name: impl AsRef, f: F) -> anyhow::Result<(T, IndexMeta)> + where + F: FnOnce(&Index) -> anyhow::Result, + { + let mut txn = self.env.write_txn()?; + let (index, _) = self.get_index_txn(&txn, &name)? + .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; + let result = f(index.as_ref()); + match result { + Ok(ret) => { + let meta = self.update_meta(&mut txn, name, |meta| meta.updated_at = Utc::now())?; + txn.commit()?; + Ok((ret, meta)) + } + Err(e) => Err(e) + } + } + + pub fn index_with_meta(&self, name: impl AsRef) -> anyhow::Result, IndexMeta)>> { + let txn = self.env.read_txn()?; + let uuid = self.index_uuid(&txn, &name)?; + match uuid { + Some(uuid) => { + let meta = self.uuid_to_index_meta.get(&txn, uuid.as_bytes())? + .with_context(|| format!("unable to retrieve metadata for index {:?}", name.as_ref()))?; + let (index, _) = self.retrieve_index(&txn, uuid)? + .with_context(|| format!("unable to retrieve index {:?}", name.as_ref()))?; + Ok(Some((index, meta))) + } + None => Ok(None), + } + } + + fn update_meta(&self, txn: &mut RwTxn, name: impl AsRef, f: impl FnOnce(&mut IndexMeta)) -> anyhow::Result { + let uuid = self.index_uuid(txn, &name)? + .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; + let mut meta = self.uuid_to_index_meta + .get(txn, uuid.as_bytes())? + .with_context(|| format!("couldn't retrieve metadata for index {:?}", name.as_ref()))?; + f(&mut meta); + self.uuid_to_index_meta.put(txn, uuid.as_bytes(), &meta)?; + Ok(meta) + } + pub fn get_or_create_index( &self, name: impl AsRef, @@ -173,7 +220,14 @@ impl IndexStore { index_store_size: u64, ) -> anyhow::Result<(Arc, Arc, IndexMeta)> { let created_at = Utc::now(); - let meta = IndexMeta { update_store_size, index_store_size, uuid: uuid.clone(), created_at }; + let updated_at = created_at; + let meta = IndexMeta { + update_store_size, + index_store_size, + uuid: uuid.clone(), + created_at, + updated_at, + }; self.name_to_uuid.put(txn, name.as_ref(), uuid.as_bytes())?; self.uuid_to_index_meta.put(txn, uuid.as_bytes(), &meta)?; @@ -318,11 +372,15 @@ mod test { let txn = store.env.read_txn().unwrap(); assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); + let created_at = Utc::now(); + let updated_at = created_at; + let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone(), - created_at: Utc::now(), + created_at, + updated_at, }; let mut txn = store.env.write_txn().unwrap(); store.uuid_to_index_meta.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); @@ -344,12 +402,16 @@ mod test { assert!(store.index(&name).unwrap().is_none()); + let created_at = Utc::now(); + let updated_at = created_at; + let uuid = Uuid::new_v4(); let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone(), - created_at: Utc::now(), + created_at, + updated_at, }; let mut txn = store.env.write_txn().unwrap(); store.name_to_uuid.put(&mut txn, &name, uuid.as_bytes()).unwrap(); diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 60a02573f..62055f1a8 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -144,6 +144,44 @@ impl IndexController for LocalIndexController { } Ok(output_meta) } + + fn update_index(&self, name: impl AsRef, index_settings: IndexSettings) -> anyhow::Result { + if index_settings.name.is_some() { + bail!("can't udpate an index name.") + } + + let (primary_key, meta) = match index_settings.primary_key { + Some(ref primary_key) => { + self.indexes + .update_index(&name, |index| { + let mut txn = index.write_txn()?; + if index.primary_key(&txn)?.is_some() { + bail!("primary key already exists.") + } + index.put_primary_key(&mut txn, primary_key)?; + txn.commit()?; + Ok(Some(primary_key.clone())) + })? + }, + None => { + let (index, meta) = self.indexes + .index_with_meta(&name)? + .with_context(|| format!("index {:?} doesn't exist.", name.as_ref()))?; + let primary_key = index + .primary_key(&index.read_txn()?)? + .map(String::from); + (primary_key, meta) + }, + }; + + Ok(IndexMetadata { + name: name.as_ref().to_string(), + uuid: meta.uuid.clone(), + created_at: meta.created_at, + updated_at: meta.updated_at, + primary_key, + }) + } } fn update_primary_key(index: impl AsRef, primary_key: impl AsRef) -> anyhow::Result<()> { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 2c8dd1226..a0f8476ff 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -10,9 +10,7 @@ use std::sync::Arc; use anyhow::Result; use chrono::{DateTime, Utc}; use milli::Index; -use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; -use serde::{Serialize, Deserialize, de::Deserializer}; -use uuid::Uuid; +use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; @@ -155,10 +153,16 @@ pub trait IndexController { /// Returns, if it exists, the `Index` with the povided name. fn index(&self, name: impl AsRef) -> anyhow::Result>>; + /// Returns the udpate status an update fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>; + + /// Returns all the udpate status for an index fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>; + /// List all the indexes fn list_indexes(&self) -> anyhow::Result>; + + fn update_index(&self, name: impl AsRef, index_settings: IndexSettings) -> anyhow::Result; } diff --git a/src/routes/index.rs b/src/routes/index.rs index d682376e3..395811774 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -94,11 +94,17 @@ struct UpdateIndexResponse { #[put("/indexes/{index_uid}", wrap = "Authentication::Private")] async fn update_index( - _data: web::Data, - _path: web::Path, - _body: web::Json, + data: web::Data, + path: web::Path, + body: web::Json, ) -> Result { - todo!() + match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.name.as_ref()) { + Ok(meta) => { + let json = serde_json::to_string(&meta).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(_) => { todo!() } + } } #[delete("/indexes/{index_uid}", wrap = "Authentication::Private")] From 4ca46b9e5f3480f234ca9c0ce2934ea38d3f72be Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 9 Feb 2021 13:25:20 +0100 Subject: [PATCH 040/522] fix bug in error message --- src/index_controller/local_index_controller/index_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 36c864772..96ef3e7cd 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -257,7 +257,7 @@ impl IndexStore { let mut txn = self.env.write_txn()?; if self.name_to_uuid.get(&txn, name.as_ref())?.is_some() { - bail!("cannot create index {:?}: an index with this name already exists.") + bail!("index {:?} already exists", name.as_ref()) } let result = self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; From 8fd9dc231c05cf1ab277f0d3a3c0b36163a074e5 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Feb 2021 17:08:37 +0100 Subject: [PATCH 041/522] implement retrieve all documents --- src/data/search.rs | 43 ++++++++++++++++++++++++++++++++++++++++-- src/routes/document.rs | 34 ++++++++++++++++++++++++++------- src/routes/index.rs | 1 - 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index d0858d704..329efc3ea 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -1,8 +1,9 @@ use std::collections::HashSet; use std::mem; use std::time::Instant; +use std::ops::RangeBounds; -use anyhow::bail; +use anyhow::{bail, Context}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; use milli::{Index, obkv_to_json, FacetCondition}; use serde::{Deserialize, Serialize}; @@ -70,7 +71,7 @@ impl SearchQuery { let highlighter = Highlighter::new(&stop_words); for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { - let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); + let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv)?; if let Some(ref attributes_to_highlight) = self.attributes_to_highlight { highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); } @@ -165,4 +166,42 @@ impl Data { None => bail!("index {:?} doesn't exists", index.as_ref()), } } + + pub fn retrieve_documents( + &self, + index: impl AsRef, + offset: usize, + count: usize, + attributes_to_retrieve: Option<&[&str]>, + ) -> anyhow::Result>> { + let index = self.index_controller + .index(&index)? + .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + let txn = index.read_txn()?; + + let mut documents = Vec::new(); + + let fields_ids_map = index.fields_ids_map(&txn)?; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .as_ref() + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let iter = index.documents.range(&txn, &(..))? + .skip(offset) + .take(count); + + for entry in iter { + let (_id, obkv) = entry?; + let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; + documents.push(object); + } + + Ok(documents) + } } diff --git a/src/routes/document.rs b/src/routes/document.rs index dcc669f85..1114ddcb3 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -12,6 +12,9 @@ use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; +const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; +const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; + macro_rules! guard_content_type { ($fn_name:ident, $guard_value:literal) => { fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { @@ -69,18 +72,35 @@ async fn delete_document( #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct BrowseQuery { - _offset: Option, - _limit: Option, - _attributes_to_retrieve: Option, + offset: Option, + limit: Option, + attributes_to_retrieve: Option, } #[get("/indexes/{index_uid}/documents", wrap = "Authentication::Public")] async fn get_all_documents( - _data: web::Data, - _path: web::Path, - _params: web::Query, + data: web::Data, + path: web::Path, + params: web::Query, ) -> Result { - todo!() + let attributes_to_retrieve = params + .attributes_to_retrieve + .as_ref() + .map(|attrs| attrs + .split(",") + .collect::>()); + + match data.retrieve_documents( + &path.index_uid, + params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), + params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), + attributes_to_retrieve.as_deref()) { + Ok(docs) => { + let json = serde_json::to_string(&docs).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(_) => { todo!() } + } } #[derive(Deserialize)] diff --git a/src/routes/index.rs b/src/routes/index.rs index d682376e3..a77f26e1f 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -32,7 +32,6 @@ async fn list_indexes(data: web::Data) -> Result Date: Tue, 9 Feb 2021 16:08:13 +0100 Subject: [PATCH 042/522] add tests --- .../local_index_controller/index_store.rs | 7 ++- src/index_controller/mod.rs | 57 ++++++++++++++++++- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 96ef3e7cd..226915016 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -133,7 +133,7 @@ impl IndexStore { } /// Use this function to perform an update on an index. - /// This function also puts a lock on what index in allowed to perform an update. + /// This function also puts a lock on what index is allowed to perform an update. pub fn update_index(&self, name: impl AsRef, f: F) -> anyhow::Result<(T, IndexMeta)> where F: FnOnce(&Index) -> anyhow::Result, @@ -167,7 +167,10 @@ impl IndexStore { } } - fn update_meta(&self, txn: &mut RwTxn, name: impl AsRef, f: impl FnOnce(&mut IndexMeta)) -> anyhow::Result { + fn update_meta(&self, txn: &mut RwTxn, name: impl AsRef, f: F) -> anyhow::Result + where + F: FnOnce(&mut IndexMeta) + { let uuid = self.index_uuid(txn, &name)? .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; let mut meta = self.uuid_to_index_meta diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index a0f8476ff..77d91575a 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -10,7 +10,9 @@ use std::sync::Arc; use anyhow::Result; use chrono::{DateTime, Utc}; use milli::Index; -use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; +use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; +use serde::{Serialize, Deserialize, de::Deserializer}; +use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; @@ -95,6 +97,7 @@ pub enum UpdateResult { Other, } +#[derive(Clone, Debug)] pub struct IndexSettings { pub name: Option, pub primary_key: Option, @@ -183,10 +186,15 @@ pub(crate) mod test { fn test_create_index_with_no_name_is_error() { crate::index_controller::test::create_index_with_no_name_is_error($controller_buider); } + + #[test] + fn test_update_index() { + crate::index_controller::test::update_index($controller_buider); + } }; } - pub(crate) fn create_and_list_indexes(controller: S) { + pub(crate) fn create_and_list_indexes(controller: impl IndexController) { let settings1 = IndexSettings { name: Some(String::from("test_index")), primary_key: None, @@ -207,11 +215,54 @@ pub(crate) mod test { assert_eq!(indexes[1].primary_key.clone().unwrap(), "foo"); } - pub(crate) fn create_index_with_no_name_is_error(controller: S) { + pub(crate) fn create_index_with_no_name_is_error(controller: impl IndexController) { let settings = IndexSettings { name: None, primary_key: None, }; assert!(controller.create_index(settings).is_err()); } + + pub(crate) fn update_index(controller: impl IndexController) { + + let settings = IndexSettings { + name: Some(String::from("test")), + primary_key: None, + }; + + assert!(controller.create_index(settings).is_ok()); + + // perform empty update returns index meta unchanged + let settings = IndexSettings { + name: None, + primary_key: None, + }; + + let result = controller.update_index("test", settings).unwrap(); + assert_eq!(result.name, "test"); + assert_eq!(result.created_at, result.updated_at); + assert!(result.primary_key.is_none()); + + // Changing the name trigger an error + let settings = IndexSettings { + name: Some(String::from("bar")), + primary_key: None, + }; + + assert!(controller.update_index("test", settings).is_err()); + + // Update primary key + let settings = IndexSettings { + name: None, + primary_key: Some(String::from("foo")), + }; + + let result = controller.update_index("test", settings.clone()).unwrap(); + assert_eq!(result.name, "test"); + assert!(result.created_at < result.updated_at); + assert_eq!(result.primary_key.unwrap(), "foo"); + + // setting the primary key again is an error + assert!(controller.update_index("test", settings).is_err()); + } } From 6766de437f0ab097224e5c8f1ceb7715f371b898 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Feb 2021 10:59:23 +0100 Subject: [PATCH 043/522] implement get document --- src/data/search.rs | 48 +++++++++++++++++-- .../local_index_controller/update_store.rs | 2 + src/routes/document.rs | 21 ++++++-- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index 329efc3ea..33e48e4fd 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -1,7 +1,6 @@ use std::collections::HashSet; use std::mem; use std::time::Instant; -use std::ops::RangeBounds; use anyhow::{bail, Context}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; @@ -171,15 +170,14 @@ impl Data { &self, index: impl AsRef, offset: usize, - count: usize, + limit: usize, attributes_to_retrieve: Option<&[&str]>, ) -> anyhow::Result>> { let index = self.index_controller .index(&index)? .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let txn = index.read_txn()?; - let mut documents = Vec::new(); + let txn = index.read_txn()?; let fields_ids_map = index.fields_ids_map(&txn)?; @@ -194,7 +192,9 @@ impl Data { let iter = index.documents.range(&txn, &(..))? .skip(offset) - .take(count); + .take(limit); + + let mut documents = Vec::new(); for entry in iter { let (_id, obkv) = entry?; @@ -204,4 +204,42 @@ impl Data { Ok(documents) } + + pub fn retrieve_document( + &self, + index: impl AsRef, + document_id: impl AsRef, + attributes_to_retrieve: Option<&[&str]>, + ) -> anyhow::Result> { + let index = self.index_controller + .index(&index)? + .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + let txn = index.read_txn()?; + + let fields_ids_map = index.fields_ids_map(&txn)?; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .as_ref() + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let internal_id = index + .external_documents_ids(&txn)? + .get(document_id.as_ref().as_bytes()) + .with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; + + let document = index.documents(&txn, std::iter::once(internal_id))? + .into_iter() + .next() + .map(|(_, d)| d); + + match document { + Some(document) => Ok(obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, document)?), + None => bail!("Document with id {} not found", document_id.as_ref()), + } + } } diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs index d4b796993..9f0c4ad7d 100644 --- a/src/index_controller/local_index_controller/update_store.rs +++ b/src/index_controller/local_index_controller/update_store.rs @@ -254,6 +254,7 @@ where /// Trying to abort an update that is currently being processed, an update /// that as already been processed or which doesn't actually exist, will /// return `None`. + #[allow(dead_code)] pub fn abort_update(&self, update_id: u64) -> heed::Result>> { let mut wtxn = self.env.write_txn()?; let key = BEU64::new(update_id); @@ -281,6 +282,7 @@ where /// Aborts all the pending updates, and not the one being currently processed. /// Returns the update metas and ids that were successfully aborted. + #[allow(dead_code)] pub fn abort_pendings(&self) -> heed::Result)>> { let mut wtxn = self.env.write_txn()?; let mut aborted_updates = Vec::new(); diff --git a/src/routes/document.rs b/src/routes/document.rs index 1114ddcb3..ac20cfff0 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -33,8 +33,8 @@ type Document = IndexMap; #[derive(Deserialize)] struct DocumentParam { - _index_uid: String, - _document_id: String, + index_uid: String, + document_id: String, } pub fn services(cfg: &mut web::ServiceConfig) { @@ -52,10 +52,21 @@ pub fn services(cfg: &mut web::ServiceConfig) { wrap = "Authentication::Public" )] async fn get_document( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() + let index = &path.index_uid; + let id = &path.document_id; + match data.retrieve_document(index, id, None) { + Ok(document) => { + let json = serde_json::to_string(&document).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + unimplemented!() + } + } } #[delete( From a8ba80965667d98a6a37aa091760348aa0ef08c7 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Feb 2021 12:03:00 +0100 Subject: [PATCH 044/522] implement clear all documents --- src/data/updates.rs | 10 ++++++++++ .../local_index_controller/mod.rs | 8 ++++++++ .../local_index_controller/update_store.rs | 2 ++ src/index_controller/mod.rs | 3 +++ src/routes/document.rs | 15 ++++++++++++--- 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 57e9d1864..d3566bb6f 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -62,6 +62,16 @@ impl Data { Ok(update.into()) } + pub async fn clear_documents( + &self, + index: impl AsRef, + ) -> anyhow::Result { + let index_controller = self.index_controller.clone(); + let index = index.as_ref().to_string(); + let update = tokio::task::spawn_blocking(move || index_controller.clear_documents(index)).await??; + Ok(update.into()) + } + #[inline] pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { self.index_controller.update_status(index, uid) diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 62055f1a8..a0c7e0805 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -182,6 +182,14 @@ impl IndexController for LocalIndexController { primary_key, }) } + + fn clear_documents(&self, index: impl AsRef) -> anyhow::Result { + let (_, update_store) = self.indexes.index(&index)? + .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + let meta = UpdateMeta::ClearDocuments; + let pending = update_store.register_update(meta, &[]).unwrap(); + Ok(pending.into()) + } } fn update_primary_key(index: impl AsRef, primary_key: impl AsRef) -> anyhow::Result<()> { diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs index d4b796993..9f0c4ad7d 100644 --- a/src/index_controller/local_index_controller/update_store.rs +++ b/src/index_controller/local_index_controller/update_store.rs @@ -254,6 +254,7 @@ where /// Trying to abort an update that is currently being processed, an update /// that as already been processed or which doesn't actually exist, will /// return `None`. + #[allow(dead_code)] pub fn abort_update(&self, update_id: u64) -> heed::Result>> { let mut wtxn = self.env.write_txn()?; let key = BEU64::new(update_id); @@ -281,6 +282,7 @@ where /// Aborts all the pending updates, and not the one being currently processed. /// Returns the update metas and ids that were successfully aborted. + #[allow(dead_code)] pub fn abort_pendings(&self) -> heed::Result)>> { let mut wtxn = self.env.write_txn()?; let mut aborted_updates = Vec::new(); diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 77d91575a..0ea71a72a 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -128,6 +128,9 @@ pub trait IndexController { data: &[u8], ) -> anyhow::Result; + /// Clear all documents in the given index. + fn clear_documents(&self, index: impl AsRef) -> anyhow::Result; + /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; diff --git a/src/routes/document.rs b/src/routes/document.rs index dcc669f85..744a2e7f1 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -169,8 +169,17 @@ async fn delete_documents( #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn clear_all_documents( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() + match data.clear_documents(&path.index_uid).await { + Ok(update) => { + let json = serde_json::to_string(&update).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + unimplemented!(); + } + } } From c317af58bcacf71f901279e2beeb734a14b9b308 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Feb 2021 17:39:14 +0100 Subject: [PATCH 045/522] implement delete document batches --- src/data/updates.rs | 11 ++++++++++ .../local_index_controller/mod.rs | 15 ++++++++++--- .../local_index_controller/update_handler.rs | 22 +++++++++++++++++++ src/index_controller/mod.rs | 5 +++++ src/routes/document.rs | 22 +++++++++++++++---- 5 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index d3566bb6f..1f3290e47 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -72,6 +72,17 @@ impl Data { Ok(update.into()) } + pub async fn delete_documents( + &self, + index: impl AsRef, + document_ids: Vec, + ) -> anyhow::Result { + let index_controller = self.index_controller.clone(); + let index = index.as_ref().to_string(); + let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; + Ok(update.into()) + } + #[inline] pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { self.index_controller.update_status(index, uid) diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index a0c7e0805..c0903a5e1 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -43,7 +43,7 @@ impl IndexController for LocalIndexController { ) -> anyhow::Result> { let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; let meta = UpdateMeta::DocumentsAddition { method, format }; - let pending = update_store.register_update(meta, data).unwrap(); + let pending = update_store.register_update(meta, data)?; Ok(pending.into()) } @@ -54,7 +54,7 @@ impl IndexController for LocalIndexController { ) -> anyhow::Result> { let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; let meta = UpdateMeta::Settings(settings); - let pending = update_store.register_update(meta, &[]).unwrap(); + let pending = update_store.register_update(meta, &[])?; Ok(pending.into()) } @@ -187,7 +187,16 @@ impl IndexController for LocalIndexController { let (_, update_store) = self.indexes.index(&index)? .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; let meta = UpdateMeta::ClearDocuments; - let pending = update_store.register_update(meta, &[]).unwrap(); + let pending = update_store.register_update(meta, &[])?; + Ok(pending.into()) + } + + fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result { + let (_, update_store) = self.indexes.index(&index)? + .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + let meta = UpdateMeta::DeleteDocuments; + let content = serde_json::to_vec(&document_ids)?; + let pending = update_store.register_update(meta, &content)?; Ok(pending.into()) } } diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index c89b9f686..505e9aff0 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -177,6 +177,27 @@ impl UpdateHandler { Err(e) => Err(e.into()) } } + + fn delete_documents( + &self, + document_ids: &[u8], + update_builder: UpdateBuilder, + ) -> anyhow::Result { + let ids: Vec = serde_json::from_slice(document_ids)?; + let mut txn = self.index.write_txn()?; + let mut builder = update_builder.delete_documents(&mut txn, &self.index)?; + + // we ignore unexisting document ids + ids.iter().for_each(|id| { builder.delete_external_id(id); }); + + match builder.execute() { + Ok(deleted) => txn + .commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } } impl HandleUpdate for UpdateHandler { @@ -194,6 +215,7 @@ impl HandleUpdate for UpdateHandler { let result = match meta.meta() { DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), ClearDocuments => self.clear_documents(update_builder), + DeleteDocuments => self.delete_documents(content, update_builder,), Settings(settings) => self.update_settings(settings, update_builder), Facets(levels) => self.update_facets(levels, update_builder), }; diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0ea71a72a..e811b6478 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -33,6 +33,7 @@ pub struct IndexMetadata { pub enum UpdateMeta { DocumentsAddition { method: IndexDocumentsMethod, format: UpdateFormat }, ClearDocuments, + DeleteDocuments, Settings(Settings), Facets(Facets), } @@ -94,6 +95,7 @@ impl Settings { #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { DocumentsAddition(DocumentAdditionResult), + DocumentDeletion { deleted: usize }, Other, } @@ -131,6 +133,9 @@ pub trait IndexController { /// Clear all documents in the given index. fn clear_documents(&self, index: impl AsRef) -> anyhow::Result; + /// Clear all documents in the given index. + fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result; + /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; diff --git a/src/routes/document.rs b/src/routes/document.rs index 744a2e7f1..67ca2a02f 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -160,11 +160,25 @@ async fn update_documents( wrap = "Authentication::Private" )] async fn delete_documents( - _data: web::Data, - _path: web::Path, - _body: web::Json>, + data: web::Data, + path: web::Path, + body: web::Json>, ) -> Result { - todo!() + let ids = body + .iter() + .map(ToString::to_string) + .collect(); + + match data.delete_documents(&path.index_uid, ids).await { + Ok(result) => { + let json = serde_json::to_string(&result).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + unimplemented!() + } + } } #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] From 28b9c158b15ad19bdfc8d6e2b0c805b0901fdd2f Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 13 Feb 2021 10:44:20 +0100 Subject: [PATCH 046/522] implement delete single document --- src/data/updates.rs | 6 ++--- .../local_index_controller/update_handler.rs | 4 +-- src/index_controller/mod.rs | 2 +- src/routes/document.rs | 25 +++++++++++++------ 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 1f3290e47..9fd2015f6 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -64,21 +64,19 @@ impl Data { pub async fn clear_documents( &self, - index: impl AsRef, + index: impl AsRef + Sync + Send + 'static, ) -> anyhow::Result { let index_controller = self.index_controller.clone(); - let index = index.as_ref().to_string(); let update = tokio::task::spawn_blocking(move || index_controller.clear_documents(index)).await??; Ok(update.into()) } pub async fn delete_documents( &self, - index: impl AsRef, + index: impl AsRef + Sync + Send + 'static, document_ids: Vec, ) -> anyhow::Result { let index_controller = self.index_controller.clone(); - let index = index.as_ref().to_string(); let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; Ok(update.into()) } diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index 505e9aff0..7b7487ffa 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -187,7 +187,7 @@ impl UpdateHandler { let mut txn = self.index.write_txn()?; let mut builder = update_builder.delete_documents(&mut txn, &self.index)?; - // we ignore unexisting document ids + // We ignore unexisting document ids ids.iter().for_each(|id| { builder.delete_external_id(id); }); match builder.execute() { @@ -215,7 +215,7 @@ impl HandleUpdate for UpdateHandler { let result = match meta.meta() { DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), ClearDocuments => self.clear_documents(update_builder), - DeleteDocuments => self.delete_documents(content, update_builder,), + DeleteDocuments => self.delete_documents(content, update_builder), Settings(settings) => self.update_settings(settings, update_builder), Facets(levels) => self.update_facets(levels, update_builder), }; diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index e811b6478..b881e3268 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -133,7 +133,7 @@ pub trait IndexController { /// Clear all documents in the given index. fn clear_documents(&self, index: impl AsRef) -> anyhow::Result; - /// Clear all documents in the given index. + /// Delete all documents in `document_ids`. fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result; /// Updates an index settings. If the index does not exist, it will be created when the update diff --git a/src/routes/document.rs b/src/routes/document.rs index 67ca2a02f..874ad722b 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -30,8 +30,8 @@ type Document = IndexMap; #[derive(Deserialize)] struct DocumentParam { - _index_uid: String, - _document_id: String, + index_uid: String, + document_id: String, } pub fn services(cfg: &mut web::ServiceConfig) { @@ -60,10 +60,19 @@ async fn get_document( wrap = "Authentication::Private" )] async fn delete_document( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() + match data.delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]).await { + Ok(result) => { + let json = serde_json::to_string(&result).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + unimplemented!() + } + } } #[derive(Deserialize)] @@ -166,10 +175,10 @@ async fn delete_documents( ) -> Result { let ids = body .iter() - .map(ToString::to_string) + .map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) .collect(); - match data.delete_documents(&path.index_uid, ids).await { + match data.delete_documents(path.index_uid.clone(), ids).await { Ok(result) => { let json = serde_json::to_string(&result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -186,7 +195,7 @@ async fn clear_all_documents( data: web::Data, path: web::Path, ) -> Result { - match data.clear_documents(&path.index_uid).await { + match data.clear_documents(path.index_uid.clone()).await { Ok(update) => { let json = serde_json::to_string(&update).unwrap(); Ok(HttpResponse::Ok().body(json)) From 8bb1b6146f4aa97b520aea7effe3a28dd32d3a12 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Feb 2021 23:02:20 +0100 Subject: [PATCH 047/522] make retrieval non blocking --- src/data/search.rs | 126 ++++++++++++++++++++++------------------- src/routes/document.rs | 11 ++-- 2 files changed, 75 insertions(+), 62 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index 33e48e4fd..b4be47c35 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -166,80 +166,92 @@ impl Data { } } - pub fn retrieve_documents( + pub async fn retrieve_documents( &self, - index: impl AsRef, + index: impl AsRef + Send + Sync + 'static, offset: usize, limit: usize, - attributes_to_retrieve: Option<&[&str]>, - ) -> anyhow::Result>> { - let index = self.index_controller - .index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + attributes_to_retrieve: Option>, + ) -> anyhow::Result>> + where + S: AsRef + Send + Sync + 'static + { + let index_controller = self.index_controller.clone(); + let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { + let index = index_controller + .index(&index)? + .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let txn = index.read_txn()?; + let txn = index.read_txn()?; - let fields_ids_map = index.fields_ids_map(&txn)?; + let fields_ids_map = index.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .as_ref() - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; - let iter = index.documents.range(&txn, &(..))? - .skip(offset) - .take(limit); + let iter = index.documents.range(&txn, &(..))? + .skip(offset) + .take(limit); - let mut documents = Vec::new(); + let mut documents = Vec::new(); - for entry in iter { - let (_id, obkv) = entry?; - let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; - documents.push(object); - } + for entry in iter { + let (_id, obkv) = entry?; + let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; + documents.push(object); + } - Ok(documents) + Ok(documents) + }).await?; + documents } - pub fn retrieve_document( + pub async fn retrieve_document( &self, - index: impl AsRef, - document_id: impl AsRef, - attributes_to_retrieve: Option<&[&str]>, - ) -> anyhow::Result> { - let index = self.index_controller - .index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let txn = index.read_txn()?; + index: impl AsRef + Sync + Send + 'static, + document_id: impl AsRef + Sync + Send + 'static, + attributes_to_retrieve: Option>, + ) -> anyhow::Result> + where + S: AsRef + Sync + Send + 'static, + { + let index_controller = self.index_controller.clone(); + let document: anyhow::Result<_> = tokio::task::spawn_blocking(move || { + let index = index_controller + .index(&index)? + .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + let txn = index.read_txn()?; - let fields_ids_map = index.fields_ids_map(&txn)?; + let fields_ids_map = index.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .as_ref() - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; - let internal_id = index - .external_documents_ids(&txn)? - .get(document_id.as_ref().as_bytes()) - .with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; + let internal_id = index + .external_documents_ids(&txn)? + .get(document_id.as_ref().as_bytes()) + .with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; - let document = index.documents(&txn, std::iter::once(internal_id))? - .into_iter() - .next() - .map(|(_, d)| d); + let document = index.documents(&txn, std::iter::once(internal_id))? + .into_iter() + .next() + .map(|(_, d)| d); - match document { - Some(document) => Ok(obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, document)?), - None => bail!("Document with id {} not found", document_id.as_ref()), - } + match document { + Some(document) => Ok(obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, document)?), + None => bail!("Document with id {} not found", document_id.as_ref()), + } + }).await?; + document } } diff --git a/src/routes/document.rs b/src/routes/document.rs index ac20cfff0..a240867dd 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -55,9 +55,9 @@ async fn get_document( data: web::Data, path: web::Path, ) -> Result { - let index = &path.index_uid; - let id = &path.document_id; - match data.retrieve_document(index, id, None) { + let index = path.index_uid.clone(); + let id = path.document_id.clone(); + match data.retrieve_document(index, id, None as Option>).await { Ok(document) => { let json = serde_json::to_string(&document).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -99,13 +99,14 @@ async fn get_all_documents( .as_ref() .map(|attrs| attrs .split(",") + .map(String::from) .collect::>()); match data.retrieve_documents( - &path.index_uid, + path.index_uid.clone(), params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), - attributes_to_retrieve.as_deref()) { + attributes_to_retrieve).await { Ok(docs) => { let json = serde_json::to_string(&docs).unwrap(); Ok(HttpResponse::Ok().body(json)) From 1eaf28f823f3df60604fe07fd5d594e2a7c70979 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 13 Feb 2021 12:22:59 +0100 Subject: [PATCH 048/522] add primary key and update documents --- src/data/updates.rs | 3 +- .../local_index_controller/mod.rs | 3 +- .../local_index_controller/update_handler.rs | 57 +++++++++++++----- src/index_controller/mod.rs | 7 ++- src/routes/document.rs | 60 +++++++++++++------ 5 files changed, 95 insertions(+), 35 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 9fd2015f6..e2bcba9d2 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -16,6 +16,7 @@ impl Data { method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, + primary_key: Option, ) -> anyhow::Result where B: Deref, @@ -47,7 +48,7 @@ impl Data { mmap = unsafe { memmap::Mmap::map(&file)? }; &mmap }; - index_controller.add_documents(index, method, format, &bytes) + index_controller.add_documents(index, method, format, &bytes, primary_key) }).await??; Ok(update.into()) } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index c0903a5e1..186ed3ddd 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -40,9 +40,10 @@ impl IndexController for LocalIndexController { method: milli::update::IndexDocumentsMethod, format: milli::update::UpdateFormat, data: &[u8], + primary_key: Option, ) -> anyhow::Result> { let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; - let meta = UpdateMeta::DocumentsAddition { method, format }; + let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; let pending = update_store.register_update(meta, data)?; Ok(pending.into()) } diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index 7b7487ffa..7102f6f50 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -1,19 +1,19 @@ +use std::collections::HashMap; use std::io; use std::sync::Arc; -use std::collections::HashMap; use anyhow::Result; use flate2::read::GzDecoder; use grenad::CompressionType; use log::info; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use milli::Index; -use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod}; use rayon::ThreadPool; -use crate::index_controller::updates::{Processing, Processed, Failed}; -use crate::index_controller::{UpdateResult, UpdateMeta, Settings, Facets}; -use crate::option::IndexerOpts; use super::update_store::HandleUpdate; +use crate::index_controller::updates::{Failed, Processed, Processing}; +use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; +use crate::option::IndexerOpts; pub struct UpdateHandler { index: Arc, @@ -70,9 +70,19 @@ impl UpdateHandler { method: IndexDocumentsMethod, content: &[u8], update_builder: UpdateBuilder, + primary_key: Option<&str>, ) -> anyhow::Result { // We must use the write transaction of the update here. let mut wtxn = self.index.write_txn()?; + + // Set the primary key if not set already, ignore if already set. + match (self.index.primary_key(&wtxn)?, primary_key) { + (None, Some(ref primary_key)) => { + self.index.put_primary_key(&mut wtxn, primary_key)?; + } + _ => (), + } + let mut builder = update_builder.index_documents(&mut wtxn, &self.index); builder.update_format(format); builder.index_documents_method(method); @@ -84,14 +94,16 @@ impl UpdateHandler { Box::new(content) as Box }; - let result = builder.execute(reader, |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + let result = builder.execute(reader, |indexing_step, update_id| { + info!("update {}: {:?}", update_id, indexing_step) + }); match result { Ok(addition_result) => wtxn .commit() .and(Ok(UpdateResult::DocumentsAddition(addition_result))) .map_err(Into::into), - Err(e) => Err(e.into()) + Err(e) => Err(e.into()), } } @@ -105,11 +117,15 @@ impl UpdateHandler { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()) + Err(e) => Err(e.into()), } } - fn update_settings(&self, settings: &Settings, update_builder: UpdateBuilder) -> anyhow::Result { + fn update_settings( + &self, + settings: &Settings, + update_builder: UpdateBuilder, + ) -> anyhow::Result { // We must use the write transaction of the update here. let mut wtxn = self.index.write_txn()?; let mut builder = update_builder.settings(&mut wtxn, &self.index); @@ -144,21 +160,22 @@ impl UpdateHandler { } } - let result = builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + let result = builder + .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); match result { Ok(()) => wtxn .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()) + Err(e) => Err(e.into()), } } fn update_facets( &self, levels: &Facets, - update_builder: UpdateBuilder + update_builder: UpdateBuilder, ) -> anyhow::Result { // We must use the write transaction of the update here. let mut wtxn = self.index.write_txn()?; @@ -174,7 +191,7 @@ impl UpdateHandler { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()) + Err(e) => Err(e.into()), } } @@ -204,7 +221,7 @@ impl HandleUpdate for UpdateHandler { fn handle_update( &mut self, meta: Processing, - content: &[u8] + content: &[u8], ) -> Result, Failed> { use UpdateMeta::*; @@ -213,7 +230,17 @@ impl HandleUpdate for UpdateHandler { let update_builder = self.update_buidler(update_id); let result = match meta.meta() { - DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), + DocumentsAddition { + method, + format, + primary_key, + } => self.update_documents( + *format, + *method, + content, + update_builder, + primary_key.as_deref(), + ), ClearDocuments => self.clear_documents(update_builder), DeleteDocuments => self.delete_documents(content, update_builder), Settings(settings) => self.update_settings(settings, update_builder), diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index b881e3268..5bdaf8737 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -31,7 +31,11 @@ pub struct IndexMetadata { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] pub enum UpdateMeta { - DocumentsAddition { method: IndexDocumentsMethod, format: UpdateFormat }, + DocumentsAddition { + method: IndexDocumentsMethod, + format: UpdateFormat, + primary_key: Option, + }, ClearDocuments, DeleteDocuments, Settings(Settings), @@ -128,6 +132,7 @@ pub trait IndexController { method: IndexDocumentsMethod, format: UpdateFormat, data: &[u8], + primary_key: Option, ) -> anyhow::Result; /// Clear all documents in the given index. diff --git a/src/routes/document.rs b/src/routes/document.rs index 9daaff6c8..0eb2bd463 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -127,17 +127,7 @@ async fn get_all_documents( #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateDocumentsQuery { - _primary_key: Option, -} - -async fn update_multiple_documents( - _data: web::Data, - _path: web::Path, - _params: web::Query, - _body: web::Json>, - _is_partial: bool, -) -> Result { - todo!() + primary_key: Option, } /// Route used when the payload type is "application/json" @@ -149,15 +139,16 @@ async fn update_multiple_documents( async fn add_documents_json( data: web::Data, path: web::Path, - _params: web::Query, + params: web::Query, body: Payload, ) -> Result { let addition_result = data .add_documents( path.into_inner().index_uid, - IndexDocumentsMethod::UpdateDocuments, + IndexDocumentsMethod::ReplaceDocuments, UpdateFormat::Json, - body + body, + params.primary_key.clone(), ).await; match addition_result { @@ -174,7 +165,7 @@ async fn add_documents_json( } -/// Default route for addign documents, this should return an error en redirect to the docuentation +/// Default route for adding documents, this should return an error and redirect to the documentation #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn add_documents_default( _data: web::Data, @@ -186,14 +177,49 @@ async fn add_documents_default( todo!() } +/// Default route for adding documents, this should return an error and redirect to the documentation #[put("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] +async fn update_documents_default( + _data: web::Data, + _path: web::Path, + _params: web::Query, + _body: web::Json>, +) -> Result { + error!("Unknown document type"); + todo!() +} + +#[put( + "/indexes/{index_uid}/documents", + wrap = "Authentication::Private", + guard = "guard_json", +)] async fn update_documents( data: web::Data, path: web::Path, params: web::Query, - body: web::Json>, + body: web::Payload, ) -> Result { - update_multiple_documents(data, path, params, body, true).await + let addition_result = data + .add_documents( + path.into_inner().index_uid, + IndexDocumentsMethod::UpdateDocuments, + UpdateFormat::Json, + body, + params.primary_key.clone(), + ).await; + + match addition_result { + Ok(update) => { + let value = serde_json::to_string(&update).unwrap(); + let response = HttpResponse::Ok().body(value); + Ok(response) + } + Err(e) => { + error!("{}", e); + todo!() + } + } } #[post( From a9e9e72840d32955277fbe5585c504651a55ca25 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Feb 2021 10:53:21 +0100 Subject: [PATCH 049/522] implement index deletion --- src/data/updates.rs | 9 +++ .../local_index_controller/index_store.rs | 63 ++++++++++++++++++- .../local_index_controller/mod.rs | 4 +- .../local_index_controller/update_store.rs | 26 ++++++-- src/routes/index.rs | 12 +++- 5 files changed, 102 insertions(+), 12 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index e2bcba9d2..8a7052ade 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -82,6 +82,15 @@ impl Data { Ok(update.into()) } + pub async fn delete_index( + &self, + index: impl AsRef + Send + Sync + 'static, + ) -> anyhow::Result<()> { + let index_controller = self.index_controller.clone(); + tokio::task::spawn_blocking(move || { index_controller.delete_index(index) }).await??; + Ok(()) + } + #[inline] pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { self.index_controller.update_status(index, uid) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 226915016..2df1675a8 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -1,3 +1,4 @@ +use std::time::Duration; use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -6,7 +7,7 @@ use anyhow::{bail, Context}; use chrono::{DateTime, Utc}; use dashmap::{DashMap, mapref::entry::Entry}; use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; -use log::error; +use log::{error, info}; use milli::Index; use rayon::ThreadPool; use serde::{Serialize, Deserialize}; @@ -90,6 +91,52 @@ impl IndexStore { }) } + pub fn delete(&self, index_uid: impl AsRef) -> anyhow::Result<()> { + // we remove the references to the index from the index map so it is not accessible anymore + let mut txn = self.env.write_txn()?; + let uuid = self.index_uuid(&txn, &index_uid)? + .with_context(|| format!("Index {:?} doesn't exist", index_uid.as_ref()))?; + self.name_to_uuid.delete(&mut txn, index_uid.as_ref())?; + self.uuid_to_index_meta.delete(&mut txn, uuid.as_bytes())?; + txn.commit()?; + // If the index was loaded, we need to close it. Since we already removed references to it + // from the index_store, the only that can still get a reference to it is the update store. + // + // First, we want to remove any pending updates from the store. + // Second, we try to get ownership on the update store so we can close it. It may take a + // couple of tries, but since the update store event loop only has a weak reference to + // itself, and we are the only other function holding a reference to it otherwise, we will + // get it eventually. + // Fourth, we request a closing of the update store. + // Fifth, we can take ownership on the index, and close it. + // Lastly, remove all the files from the file system. + let index_uid = index_uid.as_ref().to_string(); + if let Some((_, (index, updates))) = self.uuid_to_index.remove(&uuid) { + std::thread::spawn(move || { + info!("Preparing for {:?} deletion.", index_uid); + // this error is non fatal, but may delay the deletion. + if let Err(e) = updates.abort_pendings() { + error!( + "error aborting pending updates when deleting index {:?}: {}", + index_uid, + e + ); + } + let updates = get_arc_ownership_blocking(updates); + let close_event = updates.prepare_for_closing(); + close_event.wait(); + info!("closed update store for {:?}", index_uid); + + let index = get_arc_ownership_blocking(index); + let close_event = index.prepare_for_closing(); + close_event.wait(); + info!("index {:?} deleted.", index_uid); + }); + } + + Ok(()) + } + fn index_uuid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { match self.name_to_uuid.get(txn, name.as_ref())? { Some(bytes) => { @@ -299,6 +346,20 @@ impl IndexStore { } } +// Loops on an arc to get ownership on the wrapped value. This method sleeps 100ms before retrying. +fn get_arc_ownership_blocking(mut item: Arc) -> T { + loop { + match Arc::try_unwrap(item) { + Ok(item) => return item, + Err(item_arc) => { + item = item_arc; + std::thread::sleep(Duration::from_millis(100)); + continue; + } + } + } +} + fn open_or_create_database(env: &Env, name: Option<&str>) -> anyhow::Result> { match env.open_database::(name)? { Some(db) => Ok(db), diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 186ed3ddd..47de1a370 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -80,8 +80,8 @@ impl IndexController for LocalIndexController { Ok(meta) } - fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { - todo!() + fn delete_index>(&self, index_uid: S) -> anyhow::Result<()> { + self.indexes.delete(index_uid) } fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs index 9f0c4ad7d..32f734ad4 100644 --- a/src/index_controller/local_index_controller/update_store.rs +++ b/src/index_controller/local_index_controller/update_store.rs @@ -66,15 +66,23 @@ where processing, }); - let update_store_cloned = update_store.clone(); + // We need a week reference so we can take ownership on the arc later when we + // want to close the index. + let update_store_weak = Arc::downgrade(&update_store); std::thread::spawn(move || { // Block and wait for something to process. - for () in notification_receiver { + 'outer: for _ in notification_receiver { loop { - match update_store_cloned.process_pending_update(&mut update_handler) { - Ok(Some(_)) => (), - Ok(None) => break, - Err(e) => eprintln!("error while processing update: {}", e), + match update_store_weak.upgrade() { + Some(update_store) => { + match update_store.process_pending_update(&mut update_handler) { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => eprintln!("error while processing update: {}", e), + } + } + // the ownership on the arc has been taken, we need to exit + None => break 'outer, } } } @@ -83,6 +91,12 @@ where Ok(update_store) } + pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { + // We ignore this error, since that would mean the event loop is already closed. + let closing_event = self.env.prepare_for_closing(); + closing_event + } + /// Returns the new biggest id to use to store the new update. fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { let last_pending = self.pending_meta diff --git a/src/routes/index.rs b/src/routes/index.rs index 20f40069c..151202a88 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -108,10 +108,16 @@ async fn update_index( #[delete("/indexes/{index_uid}", wrap = "Authentication::Private")] async fn delete_index( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() + match data.delete_index(path.index_uid.clone()).await { + Ok(_) => Ok(HttpResponse::Ok().finish()), + Err(e) => { + error!("{}", e); + todo!() + } + } } #[derive(Deserialize)] From 5c0b5412486f46ea6e1b5a587d6fd5d44be43f14 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Feb 2021 23:32:38 +0100 Subject: [PATCH 050/522] delete db files on deletion --- .../local_index_controller/index_store.rs | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 2df1675a8..f2e00e4fe 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -99,18 +99,19 @@ impl IndexStore { self.name_to_uuid.delete(&mut txn, index_uid.as_ref())?; self.uuid_to_index_meta.delete(&mut txn, uuid.as_bytes())?; txn.commit()?; - // If the index was loaded, we need to close it. Since we already removed references to it - // from the index_store, the only that can still get a reference to it is the update store. + // If the index was loaded (i.e it is present in the uuid_to_index map), then we need to + // close it. The process goes as follow: // - // First, we want to remove any pending updates from the store. - // Second, we try to get ownership on the update store so we can close it. It may take a + // 1) We want to remove any pending updates from the store. + // 2) We try to get ownership on the update store so we can close it. It may take a // couple of tries, but since the update store event loop only has a weak reference to // itself, and we are the only other function holding a reference to it otherwise, we will // get it eventually. - // Fourth, we request a closing of the update store. - // Fifth, we can take ownership on the index, and close it. - // Lastly, remove all the files from the file system. + // 3) We request a closing of the update store. + // 4) We can take ownership on the index, and close it. + // 5) We remove all the files from the file system. let index_uid = index_uid.as_ref().to_string(); + let path = self.env.path().to_owned(); if let Some((_, (index, updates))) = self.uuid_to_index.remove(&uuid) { std::thread::spawn(move || { info!("Preparing for {:?} deletion.", index_uid); @@ -130,6 +131,18 @@ impl IndexStore { let index = get_arc_ownership_blocking(index); let close_event = index.prepare_for_closing(); close_event.wait(); + + let update_path = make_update_db_path(&path, &uuid); + let index_path = make_index_db_path(&path, &uuid); + + if let Err(e) = remove_dir_all(index_path) { + error!("error removing index {:?}: {}", index_uid, e); + } + + if let Err(e) = remove_dir_all(update_path) { + error!("error removing index {:?}: {}", index_uid, e); + } + info!("index {:?} deleted.", index_uid); }); } From aad5b789a7e2ed7c8825c8ff16b4f76efb410d0a Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Feb 2021 23:37:08 +0100 Subject: [PATCH 051/522] review edits --- .../local_index_controller/index_store.rs | 189 ++++++++++++------ .../local_index_controller/update_store.rs | 8 +- 2 files changed, 130 insertions(+), 67 deletions(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index f2e00e4fe..3fe8a3f59 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -1,21 +1,24 @@ -use std::time::Duration; use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use anyhow::{bail, Context}; use chrono::{DateTime, Utc}; -use dashmap::{DashMap, mapref::entry::Entry}; -use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; +use dashmap::{mapref::entry::Entry, DashMap}; +use heed::{ + types::{ByteSlice, SerdeJson, Str}, + Database, Env, EnvOpenOptions, RoTxn, RwTxn, +}; use log::{error, info}; use milli::Index; use rayon::ThreadPool; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::option::IndexerOpts; use super::update_handler::UpdateHandler; use super::{UpdateMeta, UpdateResult}; +use crate::option::IndexerOpts; type UpdateStore = super::update_store::UpdateStore; @@ -94,7 +97,8 @@ impl IndexStore { pub fn delete(&self, index_uid: impl AsRef) -> anyhow::Result<()> { // we remove the references to the index from the index map so it is not accessible anymore let mut txn = self.env.write_txn()?; - let uuid = self.index_uuid(&txn, &index_uid)? + let uuid = self + .index_uuid(&txn, &index_uid)? .with_context(|| format!("Index {:?} doesn't exist", index_uid.as_ref()))?; self.name_to_uuid.delete(&mut txn, index_uid.as_ref())?; self.uuid_to_index_meta.delete(&mut txn, uuid.as_bytes())?; @@ -119,8 +123,7 @@ impl IndexStore { if let Err(e) = updates.abort_pendings() { error!( "error aborting pending updates when deleting index {:?}: {}", - index_uid, - e + index_uid, e ); } let updates = get_arc_ownership_blocking(updates); @@ -156,23 +159,26 @@ impl IndexStore { let uuid = Uuid::from_slice(bytes)?; Ok(Some(uuid)) } - None => Ok(None) + None => Ok(None), } } - fn retrieve_index(&self, txn: &RoTxn, uid: Uuid) -> anyhow::Result, Arc)>> { + fn retrieve_index( + &self, + txn: &RoTxn, + uid: Uuid, + ) -> anyhow::Result, Arc)>> { match self.uuid_to_index.entry(uid.clone()) { - Entry::Vacant(entry) => { - match self.uuid_to_index_meta.get(txn, uid.as_bytes())? { - Some(meta) => { - let path = self.env.path(); - let (index, updates) = meta.open(path, self.thread_pool.clone(), &self.indexer_options)?; - entry.insert((index.clone(), updates.clone())); - Ok(Some((index, updates))) - }, - None => Ok(None) + Entry::Vacant(entry) => match self.uuid_to_index_meta.get(txn, uid.as_bytes())? { + Some(meta) => { + let path = self.env.path(); + let (index, updates) = + meta.open(path, self.thread_pool.clone(), &self.indexer_options)?; + entry.insert((index.clone(), updates.clone())); + Ok(Some((index, updates))) } - } + None => Ok(None), + }, Entry::Occupied(entry) => { let (index, updates) = entry.get(); Ok(Some((index.clone(), updates.clone()))) @@ -180,14 +186,21 @@ impl IndexStore { } } - fn get_index_txn(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { + fn get_index_txn( + &self, + txn: &RoTxn, + name: impl AsRef, + ) -> anyhow::Result, Arc)>> { match self.index_uuid(&txn, name)? { Some(uid) => self.retrieve_index(&txn, uid), None => Ok(None), } } - pub fn index(&self, name: impl AsRef) -> anyhow::Result, Arc)>> { + pub fn index( + &self, + name: impl AsRef, + ) -> anyhow::Result, Arc)>> { let txn = self.env.read_txn()?; self.get_index_txn(&txn, name) } @@ -199,7 +212,8 @@ impl IndexStore { F: FnOnce(&Index) -> anyhow::Result, { let mut txn = self.env.write_txn()?; - let (index, _) = self.get_index_txn(&txn, &name)? + let (index, _) = self + .get_index_txn(&txn, &name)? .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; let result = f(index.as_ref()); match result { @@ -208,18 +222,26 @@ impl IndexStore { txn.commit()?; Ok((ret, meta)) } - Err(e) => Err(e) + Err(e) => Err(e), } } - pub fn index_with_meta(&self, name: impl AsRef) -> anyhow::Result, IndexMeta)>> { + pub fn index_with_meta( + &self, + name: impl AsRef, + ) -> anyhow::Result, IndexMeta)>> { let txn = self.env.read_txn()?; let uuid = self.index_uuid(&txn, &name)?; match uuid { Some(uuid) => { - let meta = self.uuid_to_index_meta.get(&txn, uuid.as_bytes())? - .with_context(|| format!("unable to retrieve metadata for index {:?}", name.as_ref()))?; - let (index, _) = self.retrieve_index(&txn, uuid)? + let meta = self + .uuid_to_index_meta + .get(&txn, uuid.as_bytes())? + .with_context(|| { + format!("unable to retrieve metadata for index {:?}", name.as_ref()) + })?; + let (index, _) = self + .retrieve_index(&txn, uuid)? .with_context(|| format!("unable to retrieve index {:?}", name.as_ref()))?; Ok(Some((index, meta))) } @@ -227,13 +249,20 @@ impl IndexStore { } } - fn update_meta(&self, txn: &mut RwTxn, name: impl AsRef, f: F) -> anyhow::Result + fn update_meta( + &self, + txn: &mut RwTxn, + name: impl AsRef, + f: F, + ) -> anyhow::Result where - F: FnOnce(&mut IndexMeta) + F: FnOnce(&mut IndexMeta), { - let uuid = self.index_uuid(txn, &name)? - .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; - let mut meta = self.uuid_to_index_meta + let uuid = self + .index_uuid(txn, &name)? + .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; + let mut meta = self + .uuid_to_index_meta .get(txn, uuid.as_bytes())? .with_context(|| format!("couldn't retrieve metadata for index {:?}", name.as_ref()))?; f(&mut meta); @@ -252,7 +281,8 @@ impl IndexStore { Some(res) => Ok(res), None => { let uuid = Uuid::new_v4(); - let (index, updates, _) = self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; + let (index, updates, _) = + self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; // If we fail to commit the transaction, we must delete the database from the // file-system. if let Err(e) = txn.commit() { @@ -260,7 +290,7 @@ impl IndexStore { return Err(e)?; } Ok((index, updates)) - }, + } } } @@ -275,7 +305,8 @@ impl IndexStore { self.uuid_to_index.remove(&uuid); } - fn create_index_txn( &self, + fn create_index_txn( + &self, txn: &mut RwTxn, uuid: Uuid, name: impl AsRef, @@ -296,15 +327,17 @@ impl IndexStore { self.uuid_to_index_meta.put(txn, uuid.as_bytes(), &meta)?; let path = self.env.path(); - let (index, update_store) = match meta.open(path, self.thread_pool.clone(), &self.indexer_options) { - Ok(res) => res, - Err(e) => { - self.clean_db(uuid); - return Err(e) - } - }; + let (index, update_store) = + match meta.open(path, self.thread_pool.clone(), &self.indexer_options) { + Ok(res) => res, + Err(e) => { + self.clean_db(uuid); + return Err(e); + } + }; - self.uuid_to_index.insert(uuid, (index.clone(), update_store.clone())); + self.uuid_to_index + .insert(uuid, (index.clone(), update_store.clone())); Ok((index, update_store, meta)) } @@ -338,19 +371,24 @@ impl IndexStore { /// This method will force all the indexes to be loaded. pub fn list_indexes(&self) -> anyhow::Result)>> { let txn = self.env.read_txn()?; - let metas = self.name_to_uuid - .iter(&txn)? - .filter_map(|entry| entry - .map_err(|e| { error!("error decoding entry while listing indexes: {}", e); e }).ok()); + let metas = self.name_to_uuid.iter(&txn)?.filter_map(|entry| { + entry + .map_err(|e| { + error!("error decoding entry while listing indexes: {}", e); + e + }) + .ok() + }); let mut indexes = Vec::new(); for (name, uuid) in metas { // get index to retrieve primary key - let (index, _) = self.get_index_txn(&txn, name)? + let (index, _) = self + .get_index_txn(&txn, name)? .with_context(|| format!("could not load index {:?}", name))?; - let primary_key = index.primary_key(&index.read_txn()?)? - .map(String::from); + let primary_key = index.primary_key(&index.read_txn()?)?.map(String::from); // retieve meta - let meta = self.uuid_to_index_meta + let meta = self + .uuid_to_index_meta .get(&txn, &uuid)? .with_context(|| format!("could not retieve meta for index {:?}", name))?; indexes.push((name.to_owned(), meta, primary_key)); @@ -373,7 +411,10 @@ fn get_arc_ownership_blocking(mut item: Arc) -> T { } } -fn open_or_create_database(env: &Env, name: Option<&str>) -> anyhow::Result> { +fn open_or_create_database( + env: &Env, + name: Option<&str>, +) -> anyhow::Result> { match env.open_database::(name)? { Some(db) => Ok(db), None => Ok(env.create_database::(name)?), @@ -432,7 +473,10 @@ mod test { // insert an uuid in the the name_to_uuid_db: let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.name_to_uuid.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store + .name_to_uuid + .put(&mut txn, &name, uuid.as_bytes()) + .unwrap(); txn.commit().unwrap(); // check that the uuid is there @@ -460,7 +504,10 @@ mod test { updated_at, }; let mut txn = store.env.write_txn().unwrap(); - store.uuid_to_index_meta.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + store + .uuid_to_index_meta + .put(&mut txn, uuid.as_bytes(), &meta) + .unwrap(); txn.commit().unwrap(); // the index cache should be empty @@ -491,8 +538,14 @@ mod test { updated_at, }; let mut txn = store.env.write_txn().unwrap(); - store.name_to_uuid.put(&mut txn, &name, uuid.as_bytes()).unwrap(); - store.uuid_to_index_meta.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + store + .name_to_uuid + .put(&mut txn, &name, uuid.as_bytes()) + .unwrap(); + store + .uuid_to_index_meta + .put(&mut txn, uuid.as_bytes(), &meta) + .unwrap(); txn.commit().unwrap(); assert!(store.index(&name).unwrap().is_some()); @@ -506,13 +559,19 @@ mod test { let update_store_size = 4096 * 100; let index_store_size = 4096 * 100; - store.get_or_create_index(&name, update_store_size, index_store_size).unwrap(); + store + .get_or_create_index(&name, update_store_size, index_store_size) + .unwrap(); let txn = store.env.read_txn().unwrap(); - let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); + let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = store.uuid_to_index_meta.get(&txn, uuid.as_bytes()).unwrap().unwrap(); + let meta = store + .uuid_to_index_meta + .get(&txn, uuid.as_bytes()) + .unwrap() + .unwrap(); assert_eq!(meta.update_store_size, update_store_size); assert_eq!(meta.index_store_size, index_store_size); assert_eq!(meta.uuid, uuid); @@ -528,12 +587,18 @@ mod test { let index_store_size = 4096 * 100; let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.create_index_txn(&mut txn, uuid, name, update_store_size, index_store_size).unwrap(); + store + .create_index_txn(&mut txn, uuid, name, update_store_size, index_store_size) + .unwrap(); let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = store.uuid_to_index_meta.get(&txn, uuid.as_bytes()).unwrap().unwrap(); + let meta = store + .uuid_to_index_meta + .get(&txn, uuid.as_bytes()) + .unwrap() + .unwrap(); assert_eq!(meta.update_store_size, update_store_size); assert_eq!(meta.index_store_size, index_store_size); assert_eq!(meta.uuid, uuid); diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs index 32f734ad4..b025ff090 100644 --- a/src/index_controller/local_index_controller/update_store.rs +++ b/src/index_controller/local_index_controller/update_store.rs @@ -66,7 +66,7 @@ where processing, }); - // We need a week reference so we can take ownership on the arc later when we + // We need a weak reference so we can take ownership on the arc later when we // want to close the index. let update_store_weak = Arc::downgrade(&update_store); std::thread::spawn(move || { @@ -81,7 +81,7 @@ where Err(e) => eprintln!("error while processing update: {}", e), } } - // the ownership on the arc has been taken, we need to exit + // the ownership on the arc has been taken, we need to exit. None => break 'outer, } } @@ -92,9 +92,7 @@ where } pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { - // We ignore this error, since that would mean the event loop is already closed. - let closing_event = self.env.prepare_for_closing(); - closing_event + self.env.prepare_for_closing() } /// Returns the new biggest id to use to store the new update. From 1823fa18c9090ab61e39da293e4b01a1f25b80d9 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 16 Feb 2021 15:26:13 +0100 Subject: [PATCH 052/522] add basic error handling --- src/routes/document.rs | 22 +++++++++------------- src/routes/index.rs | 26 ++++++++++++-------------- src/routes/search.rs | 4 +--- 3 files changed, 22 insertions(+), 30 deletions(-) diff --git a/src/routes/document.rs b/src/routes/document.rs index 0eb2bd463..43d88e4b9 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -63,8 +63,7 @@ async fn get_document( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("{}", e); - unimplemented!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -83,8 +82,7 @@ async fn delete_document( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("{}", e); - unimplemented!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -120,7 +118,9 @@ async fn get_all_documents( let json = serde_json::to_string(&docs).unwrap(); Ok(HttpResponse::Ok().body(json)) } - Err(_) => { todo!() } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } } } @@ -158,8 +158,7 @@ async fn add_documents_json( Ok(response) } Err(e) => { - error!("{}", e); - todo!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -216,8 +215,7 @@ async fn update_documents( Ok(response) } Err(e) => { - error!("{}", e); - todo!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -242,8 +240,7 @@ async fn delete_documents( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("{}", e); - unimplemented!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -259,8 +256,7 @@ async fn clear_all_documents( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("{}", e); - unimplemented!(); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } diff --git a/src/routes/index.rs b/src/routes/index.rs index 151202a88..62684ddaa 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -1,7 +1,6 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; -use log::error; use serde::{Deserialize, Serialize}; use crate::Data; @@ -28,8 +27,7 @@ async fn list_indexes(data: web::Data) -> Result { - error!("error listing indexes: {}", e); - unimplemented!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -45,7 +43,8 @@ async fn get_index( Ok(HttpResponse::Ok().body(json)) } None => { - unimplemented!() + let e = format!("Index {:?} doesn't exist.", path.index_uid); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -68,8 +67,7 @@ async fn create_index( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("error creating index: {}", e); - unimplemented!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -102,7 +100,9 @@ async fn update_index( let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) } - Err(_) => { todo!() } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } } } @@ -114,8 +114,7 @@ async fn delete_index( match data.delete_index(path.index_uid.clone()).await { Ok(_) => Ok(HttpResponse::Ok().finish()), Err(e) => { - error!("{}", e); - todo!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -141,11 +140,11 @@ async fn get_update_status( Ok(HttpResponse::Ok().body(json)) } Ok(None) => { - todo!() + let e = format!("udpate {} for index {:?} doesn't exists.", path.update_id, path.index_uid); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } Err(e) => { - error!("{}", e); - todo!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -162,8 +161,7 @@ async fn get_all_updates_status( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("{}", e); - todo!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } diff --git a/src/routes/search.rs b/src/routes/search.rs index 967065687..715d40426 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -1,5 +1,4 @@ use actix_web::{get, post, web, HttpResponse}; -use log::error; use crate::error::ResponseError; use crate::helpers::Authentication; @@ -49,8 +48,7 @@ async fn search_with_post( Ok(HttpResponse::Ok().body(docs)) } Err(e) => { - error!("{}", e); - todo!() + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } From bead4075d8bbc9913b33d66aacd386251dd2e6c5 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 16 Feb 2021 15:28:19 +0100 Subject: [PATCH 053/522] implement list api keys --- src/routes/key.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/routes/key.rs b/src/routes/key.rs index be089beb2..7fb6ae00a 100644 --- a/src/routes/key.rs +++ b/src/routes/key.rs @@ -18,5 +18,9 @@ struct KeysResponse { #[get("/keys", wrap = "Authentication::Admin")] async fn list(_data: web::Data) -> HttpResponse { - todo!() + let api_keys = data.api_keys.clone(); + HttpResponse::Ok().json(KeysResponse { + private: api_keys.private, + public: api_keys.public, + }) } From e1253b696982c198a29a316c9945d9ef145eb51e Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 16 Feb 2021 15:54:07 +0100 Subject: [PATCH 054/522] enable search with get route --- src/data/mod.rs | 4 +- src/data/search.rs | 2 +- src/routes/key.rs | 2 +- src/routes/search.rs | 103 ++++++++++++++++++++++++++++++++++--------- 4 files changed, 86 insertions(+), 25 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 8c512895c..656d19880 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -1,7 +1,7 @@ mod search; mod updates; -pub use search::{SearchQuery, SearchResult}; +pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; use std::fs::create_dir_all; use std::ops::Deref; @@ -28,7 +28,7 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { pub index_controller: Arc, - api_keys: ApiKeys, + pub api_keys: ApiKeys, options: Opt, } diff --git a/src/data/search.rs b/src/data/search.rs index b4be47c35..461cfb644 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -11,7 +11,7 @@ use serde_json::{Value, Map}; use crate::index_controller::IndexController; use super::Data; -const DEFAULT_SEARCH_LIMIT: usize = 20; +pub const DEFAULT_SEARCH_LIMIT: usize = 20; const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } diff --git a/src/routes/key.rs b/src/routes/key.rs index 7fb6ae00a..a0cbaccc3 100644 --- a/src/routes/key.rs +++ b/src/routes/key.rs @@ -17,7 +17,7 @@ struct KeysResponse { } #[get("/keys", wrap = "Authentication::Admin")] -async fn list(_data: web::Data) -> HttpResponse { +async fn list(data: web::Data) -> HttpResponse { let api_keys = data.api_keys.clone(); HttpResponse::Ok().json(KeysResponse { private: api_keys.private, diff --git a/src/routes/search.rs b/src/routes/search.rs index 715d40426..a9e87d1a6 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -1,38 +1,99 @@ -use actix_web::{get, post, web, HttpResponse}; +use std::collections::HashSet; +use std::convert::{TryFrom, TryInto}; +use actix_web::{get, post, web, HttpResponse}; +use serde::Deserialize; + +use crate::data::{SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; use crate::Data; -use crate::data::SearchQuery; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(search_with_post).service(search_with_url_query); } -//#[derive(Serialize, Deserialize)] -//#[serde(rename_all = "camelCase", deny_unknown_fields)] -//pub struct SearchQuery { - //q: Option, - //offset: Option, - //limit: Option, - //attributes_to_retrieve: Option, - //attributes_to_crop: Option, - //crop_length: Option, - //attributes_to_highlight: Option, - //filters: Option, - //matches: Option, - //facet_filters: Option, - //facets_distribution: Option, -//} +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchQueryGet { + q: Option, + offset: Option, + limit: Option, + attributes_to_retrieve: Option, + attributes_to_crop: Option, + crop_length: Option, + attributes_to_highlight: Option, + filters: Option, + matches: Option, + facet_filters: Option, + facets_distribution: Option, +} + +impl TryFrom for SearchQuery { + type Error = anyhow::Error; + + fn try_from(other: SearchQueryGet) -> anyhow::Result { + let attributes_to_retrieve = other + .attributes_to_retrieve + .map(|attrs| attrs.split(",").map(String::from).collect::>()); + + let attributes_to_crop = other + .attributes_to_crop + .map(|attrs| attrs.split(",").map(String::from).collect::>()); + + let attributes_to_highlight = other + .attributes_to_highlight + .map(|attrs| attrs.split(",").map(String::from).collect::>()); + + let facets_distribution = other + .facets_distribution + .map(|attrs| attrs.split(",").map(String::from).collect::>()); + + let facet_filters = match other.facet_filters { + Some(ref f) => Some(serde_json::from_str(f)?), + None => None, + }; + + Ok(Self { + q: other.q, + offset: other.offset, + limit: other.limit.unwrap_or(DEFAULT_SEARCH_LIMIT), + attributes_to_retrieve, + attributes_to_crop, + crop_length: other.crop_length, + attributes_to_highlight, + filters: other.filters, + matches: other.matches, + facet_filters, + facets_distribution, + facet_condition: None, + }) + } +} #[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")] async fn search_with_url_query( - _data: web::Data, - _path: web::Path, - _params: web::Query, + data: web::Data, + path: web::Path, + params: web::Query, ) -> Result { - todo!() + let query: SearchQuery = match params.into_inner().try_into() { + Ok(q) => q, + Err(e) => { + return Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + }; + let search_result = data.search(&path.index_uid, query); + match search_result { + Ok(docs) => { + let docs = serde_json::to_string(&docs).unwrap(); + Ok(HttpResponse::Ok().body(docs)) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } #[post("/indexes/{index_uid}/search", wrap = "Authentication::Public")] From 4fe90a1a1c1aa53389f64346e64d9e758a65d43b Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 16 Feb 2021 16:17:28 +0100 Subject: [PATCH 055/522] fix attributes to retrieve in search --- src/data/search.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index 461cfb644..68e8aa193 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -38,7 +38,7 @@ impl SearchQuery { pub fn perform(&self, index: impl AsRef) -> anyhow::Result{ let index = index.as_ref(); let before_search = Instant::now(); - let rtxn = index.read_txn().unwrap(); + let rtxn = index.read_txn()?; let mut search = index.search(&rtxn); @@ -59,18 +59,31 @@ impl SearchQuery { let milli::SearchResult { documents_ids, found_words, candidates } = search.execute()?; let mut documents = Vec::new(); - let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + let fields_ids_map = index.fields_ids_map(&rtxn)?; - let displayed_fields = match index.displayed_fields_ids(&rtxn).unwrap() { - Some(fields) => fields, - None => fields_ids_map.iter().map(|(id, _)| id).collect(), + let displayed_fields_ids = index.displayed_fields_ids(&rtxn)?; + + let attributes_to_retrieve_ids = match self.attributes_to_retrieve { + Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, + Some(ref attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>() + .into(), + None => None, + }; + + let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { + (_, Some(ids)) => ids, + (Some(ids), None) => ids, + (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), }; let stop_words = fst::Set::default(); let highlighter = Highlighter::new(&stop_words); - for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { - let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv)?; + for (_id, obkv) in index.documents(&rtxn, documents_ids)? { + let mut object = obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv)?; if let Some(ref attributes_to_highlight) = self.attributes_to_highlight { highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); } From 91d6e90d5d0dfeb0633195887b09a97c4459fc37 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 16 Feb 2021 18:21:16 +0100 Subject: [PATCH 056/522] enable faceted searches --- Cargo.lock | 47 +-------------- Cargo.toml | 3 +- src/data/search.rs | 139 ++++++++++++++++++++++++++++++++----------- src/routes/search.rs | 1 - 4 files changed, 106 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b8128118..7e4054d87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,6 @@ dependencies = [ "atty", "bitflags", "strsim", - "term_size", "textwrap", "unicode-width", "vec_map", @@ -1634,6 +1633,7 @@ dependencies = [ "chrono", "crossbeam-channel", "dashmap", + "either", "env_logger 0.8.2", "flate2", "fst", @@ -1720,7 +1720,6 @@ version = "0.1.0" dependencies = [ "anyhow", "bstr", - "byte-unit", "byteorder", "crossbeam-channel", "csv", @@ -1732,13 +1731,11 @@ dependencies = [ "heed", "human_format", "itertools 0.9.0", - "jemallocator", "levenshtein_automata", "linked-hash-map", "log", "meilisearch-tokenizer", "memmap", - "near-proximity", "num-traits", "obkv", "once_cell", @@ -1747,15 +1744,11 @@ dependencies = [ "pest_derive", "rayon", "regex", - "ringtail", "roaring", "serde", "serde_json", - "slice-group-by", "smallstr", "smallvec", - "stderrlog", - "structopt", "tempfile", "uuid", ] @@ -1850,14 +1843,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "near-proximity" -version = "0.1.0" -source = "git+https://github.com/Kerollmops/plane-sweep-proximity?rev=6608205#66082058537f6fe7709adc4690048d62f3c0e9b7" -dependencies = [ - "tinyvec", -] - [[package]] name = "net2" version = "0.2.37" @@ -2435,12 +2420,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "ringtail" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21215c1b9d8f7832b433255bd9eea3e2779aa55b21b2f8e13aad62c74749b237" - [[package]] name = "roaring" version = "0.6.4" @@ -2737,19 +2716,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "stderrlog" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02f316286ae558d83acc93dd81eaba096e746987a7961d4a9ae026842bae67f" -dependencies = [ - "atty", - "chrono", - "log", - "termcolor", - "thread_local", -] - [[package]] name = "stdweb" version = "0.4.20" @@ -2897,16 +2863,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "term_size" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" -dependencies = [ - "libc", - "winapi 0.3.9", -] - [[package]] name = "termcolor" version = "1.1.2" @@ -2922,7 +2878,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" dependencies = [ - "term_size", "unicode-width", ] diff --git a/Cargo.toml b/Cargo.toml index 4668a6897..993c80946 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { path = "../milli" } +milli = { path = "../milli/milli" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" @@ -57,6 +57,7 @@ tokio = { version = "0.2", features = ["full"] } dashmap = "4.0.2" uuid = "0.8.2" itertools = "0.10.0" +either = "1.6.1" [dependencies.sentry] default-features = false diff --git a/src/data/search.rs b/src/data/search.rs index 68e8aa193..391cc1960 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -3,17 +3,21 @@ use std::mem; use std::time::Instant; use anyhow::{bail, Context}; +use either::Either; +use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{Index, obkv_to_json, FacetCondition}; +use milli::{obkv_to_json, FacetCondition, Index}; use serde::{Deserialize, Serialize}; -use serde_json::{Value, Map}; +use serde_json::{Map, Value}; -use crate::index_controller::IndexController; use super::Data; +use crate::index_controller::IndexController; pub const DEFAULT_SEARCH_LIMIT: usize = 20; -const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } +const fn default_search_limit() -> usize { + DEFAULT_SEARCH_LIMIT +} #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -31,11 +35,10 @@ pub struct SearchQuery { pub matches: Option, pub facet_filters: Option, pub facets_distribution: Option>, - pub facet_condition: Option, } impl SearchQuery { - pub fn perform(&self, index: impl AsRef) -> anyhow::Result{ + pub fn perform(&self, index: impl AsRef) -> anyhow::Result { let index = index.as_ref(); let before_search = Instant::now(); let rtxn = index.read_txn()?; @@ -49,14 +52,17 @@ impl SearchQuery { search.limit(self.limit); search.offset(self.offset.unwrap_or_default()); - if let Some(ref condition) = self.facet_condition { - if !condition.trim().is_empty() { - let condition = FacetCondition::from_str(&rtxn, &index, &condition)?; - search.facet_condition(condition); + if let Some(ref facets) = self.facet_filters { + if let Some(facets) = parse_facets(facets, index, &rtxn)? { + search.facet_condition(facets); } } - let milli::SearchResult { documents_ids, found_words, candidates } = search.execute()?; + let milli::SearchResult { + documents_ids, + found_words, + candidates, + } = search.execute()?; let mut documents = Vec::new(); let fields_ids_map = index.fields_ids_map(&rtxn)?; @@ -66,14 +72,14 @@ impl SearchQuery { let attributes_to_retrieve_ids = match self.attributes_to_retrieve { Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>() + .into(), None => None, }; - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { + let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { (_, Some(ids)) => ids, (Some(ids), None) => ids, (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), @@ -133,25 +139,31 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { for (word, token) in analyzed.reconstruct() { if token.is_word() { let to_highlight = words_to_highlight.contains(token.text()); - if to_highlight { string.push_str("") } + if to_highlight { + string.push_str("") + } string.push_str(word); - if to_highlight { string.push_str("") } + if to_highlight { + string.push_str("") + } } else { string.push_str(word); } } Value::String(string) - }, - Value::Array(values) => { - Value::Array(values.into_iter() + } + Value::Array(values) => Value::Array( + values + .into_iter() .map(|v| self.highlight_value(v, words_to_highlight)) - .collect()) - }, - Value::Object(object) => { - Value::Object(object.into_iter() + .collect(), + ), + Value::Object(object) => Value::Object( + object + .into_iter() .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) - .collect()) - }, + .collect(), + ), } } @@ -172,7 +184,11 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } impl Data { - pub fn search>(&self, index: S, search_query: SearchQuery) -> anyhow::Result { + pub fn search>( + &self, + index: S, + search_query: SearchQuery, + ) -> anyhow::Result { match self.index_controller.index(&index)? { Some(index) => Ok(search_query.perform(index)?), None => bail!("index {:?} doesn't exists", index.as_ref()), @@ -187,7 +203,7 @@ impl Data { attributes_to_retrieve: Option>, ) -> anyhow::Result>> where - S: AsRef + Send + Sync + 'static + S: AsRef + Send + Sync + 'static, { let index_controller = self.index_controller.clone(); let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { @@ -207,9 +223,7 @@ impl Data { None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; - let iter = index.documents.range(&txn, &(..))? - .skip(offset) - .take(limit); + let iter = index.documents.range(&txn, &(..))?.skip(offset).take(limit); let mut documents = Vec::new(); @@ -220,7 +234,8 @@ impl Data { } Ok(documents) - }).await?; + }) + .await?; documents } @@ -255,16 +270,68 @@ impl Data { .get(document_id.as_ref().as_bytes()) .with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; - let document = index.documents(&txn, std::iter::once(internal_id))? + let document = index + .documents(&txn, std::iter::once(internal_id))? .into_iter() .next() .map(|(_, d)| d); match document { - Some(document) => Ok(obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, document)?), + Some(document) => Ok(obkv_to_json( + &attributes_to_retrieve_ids, + &fields_ids_map, + document, + )?), None => bail!("Document with id {} not found", document_id.as_ref()), } - }).await?; + }) + .await?; document } } + +fn parse_facets_array( + txn: &RoTxn, + index: &Index, + arr: &Vec, +) -> anyhow::Result> { + let mut ands = Vec::new(); + for value in arr { + match value { + Value::String(s) => ands.push(Either::Right(s.clone())), + Value::Array(arr) => { + let mut ors = Vec::new(); + for value in arr { + match value { + Value::String(s) => ors.push(s.clone()), + v => bail!("Invalid facet expression, expected String, found: {:?}", v), + } + } + ands.push(Either::Left(ors)); + } + v => bail!( + "Invalid facet expression, expected String or [String], found: {:?}", + v + ), + } + } + + FacetCondition::from_array(txn, index, ands) +} + +fn parse_facets( + facets: &Value, + index: &Index, + txn: &RoTxn, +) -> anyhow::Result> { + match facets { + // Disabled for now + //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), + Value::Array(arr) => parse_facets_array(txn, index, arr), + v => bail!( + "Invalid facet expression, expected Array, found: {:?}", + v + ), + } +} + diff --git a/src/routes/search.rs b/src/routes/search.rs index a9e87d1a6..794c8ac74 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -67,7 +67,6 @@ impl TryFrom for SearchQuery { matches: other.matches, facet_filters, facets_distribution, - facet_condition: None, }) } } From ae9a41a19f537cdbdb44833e5eec8f137db514fd Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 17 Feb 2021 12:16:54 +0100 Subject: [PATCH 057/522] fix error message when empty payload --- Cargo.lock | 46 ------------------- Cargo.toml | 2 +- src/data/updates.rs | 1 - .../local_index_controller/update_handler.rs | 2 +- 4 files changed, 2 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b8128118..bffae3f11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,6 @@ dependencies = [ "atty", "bitflags", "strsim", - "term_size", "textwrap", "unicode-width", "vec_map", @@ -1720,7 +1719,6 @@ version = "0.1.0" dependencies = [ "anyhow", "bstr", - "byte-unit", "byteorder", "crossbeam-channel", "csv", @@ -1732,13 +1730,11 @@ dependencies = [ "heed", "human_format", "itertools 0.9.0", - "jemallocator", "levenshtein_automata", "linked-hash-map", "log", "meilisearch-tokenizer", "memmap", - "near-proximity", "num-traits", "obkv", "once_cell", @@ -1747,15 +1743,11 @@ dependencies = [ "pest_derive", "rayon", "regex", - "ringtail", "roaring", "serde", "serde_json", - "slice-group-by", "smallstr", "smallvec", - "stderrlog", - "structopt", "tempfile", "uuid", ] @@ -1850,14 +1842,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "near-proximity" -version = "0.1.0" -source = "git+https://github.com/Kerollmops/plane-sweep-proximity?rev=6608205#66082058537f6fe7709adc4690048d62f3c0e9b7" -dependencies = [ - "tinyvec", -] - [[package]] name = "net2" version = "0.2.37" @@ -2435,12 +2419,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "ringtail" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21215c1b9d8f7832b433255bd9eea3e2779aa55b21b2f8e13aad62c74749b237" - [[package]] name = "roaring" version = "0.6.4" @@ -2737,19 +2715,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "stderrlog" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02f316286ae558d83acc93dd81eaba096e746987a7961d4a9ae026842bae67f" -dependencies = [ - "atty", - "chrono", - "log", - "termcolor", - "thread_local", -] - [[package]] name = "stdweb" version = "0.4.20" @@ -2897,16 +2862,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "term_size" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" -dependencies = [ - "libc", - "winapi 0.3.9", -] - [[package]] name = "termcolor" version = "1.1.2" @@ -2922,7 +2877,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" dependencies = [ - "term_size", "unicode-width", ] diff --git a/Cargo.toml b/Cargo.toml index 4668a6897..d6c4ae396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { path = "../milli" } +milli = { path = "../milli/milli" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" diff --git a/src/data/updates.rs b/src/data/updates.rs index 8a7052ade..92d682b1d 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -38,7 +38,6 @@ impl Data { file.sync_all().await?; let file = file.into_std().await; - let index_controller = self.index_controller.clone(); let update = tokio::task::spawn_blocking(move ||{ let mmap; diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index 7102f6f50..5781a2806 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -88,7 +88,7 @@ impl UpdateHandler { builder.index_documents_method(method); let gzipped = true; - let reader = if gzipped { + let reader = if gzipped && !content.is_empty() { Box::new(GzDecoder::new(content)) } else { Box::new(content) as Box From 999758f7a1cfd2c233a9eed38564216bdb517693 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 17 Feb 2021 14:52:30 +0100 Subject: [PATCH 058/522] facets distribution --- src/data/search.rs | 24 ++++++++++++++++++++---- src/routes/search.rs | 8 ++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index 391cc1960..f26730fcf 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashSet, BTreeMap}; use std::mem; use std::time::Instant; @@ -6,7 +6,7 @@ use anyhow::{bail, Context}; use either::Either; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{obkv_to_json, FacetCondition, Index}; +use milli::{obkv_to_json, FacetCondition, Index, facet::FacetValue}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -34,7 +34,7 @@ pub struct SearchQuery { pub filters: Option, pub matches: Option, pub facet_filters: Option, - pub facets_distribution: Option>, + pub facet_distributions: Option>, } impl SearchQuery { @@ -96,13 +96,27 @@ impl SearchQuery { documents.push(object); } + let nb_hits = candidates.len(); + + let facet_distributions = match self.facet_distributions { + Some(ref fields) => { + let mut facet_distribution = index.facets_distribution(&rtxn); + if fields.iter().all(|f| f != "*") { + facet_distribution.facets(fields); + } + Some(facet_distribution.candidates(candidates).execute()?) + } + None => None, + }; + Ok(SearchResult { hits: documents, - nb_hits: candidates.len(), + nb_hits, query: self.q.clone().unwrap_or_default(), limit: self.limit, offset: self.offset.unwrap_or_default(), processing_time_ms: before_search.elapsed().as_millis(), + facet_distributions, }) } } @@ -116,6 +130,8 @@ pub struct SearchResult { limit: usize, offset: usize, processing_time_ms: u128, + #[serde(skip_serializing_if = "Option::is_none")] + facet_distributions: Option>>, } struct Highlighter<'a, A> { diff --git a/src/routes/search.rs b/src/routes/search.rs index 794c8ac74..7919c5412 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -27,7 +27,7 @@ pub struct SearchQueryGet { filters: Option, matches: Option, facet_filters: Option, - facets_distribution: Option, + facet_distributions: Option, } impl TryFrom for SearchQuery { @@ -46,8 +46,8 @@ impl TryFrom for SearchQuery { .attributes_to_highlight .map(|attrs| attrs.split(",").map(String::from).collect::>()); - let facets_distribution = other - .facets_distribution + let facet_distributions = other + .facet_distributions .map(|attrs| attrs.split(",").map(String::from).collect::>()); let facet_filters = match other.facet_filters { @@ -66,7 +66,7 @@ impl TryFrom for SearchQuery { filters: other.filters, matches: other.matches, facet_filters, - facets_distribution, + facet_distributions, }) } } From 588add8beca74004cbc9270760b8bd179aa833f3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Feb 2021 17:48:37 +0100 Subject: [PATCH 059/522] rename update fields to camel case --- src/data/mod.rs | 2 +- .../local_index_controller/mod.rs | 18 +++++++++--------- src/index_controller/mod.rs | 10 +++++----- src/index_controller/updates.rs | 3 ++- src/option.rs | 1 - src/routes/index.rs | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 656d19880..ed5ce4952 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -122,7 +122,7 @@ impl Data { Ok(self .list_indexes()? .into_iter() - .find(|i| i.name == name.as_ref())) + .find(|i| i.uid == name.as_ref())) } pub fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 47de1a370..14efe42c7 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -70,7 +70,7 @@ impl IndexController for LocalIndexController { } let meta = IndexMetadata { - name: index_name, + uid: index_name, uuid: meta.uuid.clone(), created_at: meta.created_at, updated_at: meta.created_at, @@ -124,18 +124,18 @@ impl IndexController for LocalIndexController { fn list_indexes(&self) -> anyhow::Result> { let metas = self.indexes.list_indexes()?; let mut output_meta = Vec::new(); - for (name, meta, primary_key) in metas { + for (uid, meta, primary_key) in metas { let created_at = meta.created_at; let uuid = meta.uuid; let updated_at = self - .all_update_status(&name)? + .all_update_status(&uid)? .iter() .filter_map(|u| u.processed().map(|u| u.processed_at)) .max() .unwrap_or(created_at); let index_meta = IndexMetadata { - name, + uid, created_at, updated_at, uuid, @@ -146,7 +146,7 @@ impl IndexController for LocalIndexController { Ok(output_meta) } - fn update_index(&self, name: impl AsRef, index_settings: IndexSettings) -> anyhow::Result { + fn update_index(&self, uid: impl AsRef, index_settings: IndexSettings) -> anyhow::Result { if index_settings.name.is_some() { bail!("can't udpate an index name.") } @@ -154,7 +154,7 @@ impl IndexController for LocalIndexController { let (primary_key, meta) = match index_settings.primary_key { Some(ref primary_key) => { self.indexes - .update_index(&name, |index| { + .update_index(&uid, |index| { let mut txn = index.write_txn()?; if index.primary_key(&txn)?.is_some() { bail!("primary key already exists.") @@ -166,8 +166,8 @@ impl IndexController for LocalIndexController { }, None => { let (index, meta) = self.indexes - .index_with_meta(&name)? - .with_context(|| format!("index {:?} doesn't exist.", name.as_ref()))?; + .index_with_meta(&uid)? + .with_context(|| format!("index {:?} doesn't exist.", uid.as_ref()))?; let primary_key = index .primary_key(&index.read_txn()?)? .map(String::from); @@ -176,7 +176,7 @@ impl IndexController for LocalIndexController { }; Ok(IndexMetadata { - name: name.as_ref().to_string(), + uid: uid.as_ref().to_string(), uuid: meta.uuid.clone(), created_at: meta.created_at, updated_at: meta.updated_at, diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 5bdaf8737..b20e43749 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -21,7 +21,7 @@ pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { - pub name: String, + pub uid: String, uuid: Uuid, created_at: DateTime, updated_at: DateTime, @@ -223,8 +223,8 @@ pub(crate) mod test { let indexes = controller.list_indexes().unwrap(); assert_eq!(indexes.len(), 2); - assert_eq!(indexes[0].name, "test_index"); - assert_eq!(indexes[1].name, "test_index2"); + assert_eq!(indexes[0].uid, "test_index"); + assert_eq!(indexes[1].uid, "test_index2"); assert_eq!(indexes[1].primary_key.clone().unwrap(), "foo"); } @@ -252,7 +252,7 @@ pub(crate) mod test { }; let result = controller.update_index("test", settings).unwrap(); - assert_eq!(result.name, "test"); + assert_eq!(result.uid, "test"); assert_eq!(result.created_at, result.updated_at); assert!(result.primary_key.is_none()); @@ -271,7 +271,7 @@ pub(crate) mod test { }; let result = controller.update_index("test", settings.clone()).unwrap(); - assert_eq!(result.name, "test"); + assert_eq!(result.uid, "test"); assert!(result.created_at < result.updated_at); assert_eq!(result.primary_key.unwrap(), "foo"); diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs index 900987ba6..b8f1e31f4 100644 --- a/src/index_controller/updates.rs +++ b/src/index_controller/updates.rs @@ -2,6 +2,7 @@ use chrono::{Utc, DateTime}; use serde::{Serialize, Deserialize}; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] pub struct Pending { pub update_id: u64, pub meta: M, @@ -115,7 +116,7 @@ impl Failed { } #[derive(Debug, PartialEq, Eq, Hash, Serialize)] -#[serde(tag = "status")] +#[serde(tag = "status", rename_all = "camelCase")] pub enum UpdateStatus { Processing(Processing), Pending(Pending), diff --git a/src/option.rs b/src/option.rs index 38f99880e..c88110ccd 100644 --- a/src/option.rs +++ b/src/option.rs @@ -64,7 +64,6 @@ pub struct IndexerOpts { pub indexing_jobs: Option, } -#[cfg(test)] impl Default for IndexerOpts { fn default() -> Self { Self { diff --git a/src/routes/index.rs b/src/routes/index.rs index 62684ddaa..2c478432b 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -52,7 +52,7 @@ async fn get_index( #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct IndexCreateRequest { - name: String, + uid: String, primary_key: Option, } @@ -61,7 +61,7 @@ async fn create_index( data: web::Data, body: web::Json, ) -> Result { - match data.create_index(&body.name, body.primary_key.clone()) { + match data.create_index(&body.uid, body.primary_key.clone()) { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) From 72eed0e369d87dba70e52a010b4018b0186e02f1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Feb 2021 19:50:52 +0100 Subject: [PATCH 060/522] test create index --- tests/common/index.rs | 27 ++ tests/common/mod.rs | 30 ++ tests/common/server.rs | 537 ++++++++++++++++++++++++++++++++++++ tests/common/service.rs | 39 +++ tests/index/create_index.rs | 48 ++++ tests/index/mod.rs | 2 + tests/integration.rs | 3 + tests/search/mod.rs | 3 + 8 files changed, 689 insertions(+) create mode 100644 tests/common/index.rs create mode 100644 tests/common/mod.rs create mode 100644 tests/common/server.rs create mode 100644 tests/common/service.rs create mode 100644 tests/index/create_index.rs create mode 100644 tests/index/mod.rs create mode 100644 tests/integration.rs create mode 100644 tests/search/mod.rs diff --git a/tests/common/index.rs b/tests/common/index.rs new file mode 100644 index 000000000..69b383b4a --- /dev/null +++ b/tests/common/index.rs @@ -0,0 +1,27 @@ +use actix_web::http::StatusCode; +use serde_json::{json, Value}; + +use super::service::Service; + +pub struct Index<'a> { + pub uid: String, + pub service: &'a Service, +} + +impl Index<'_> { + pub async fn get(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}", self.uid); + self.service.get(url).await + } + + pub async fn create<'a>( + &'a self, + primary_key: Option<&str>, + ) -> (Value, StatusCode) { + let body = json!({ + "uid": self.uid, + "primaryKey": primary_key, + }); + self.service.post("/indexes", body).await + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 000000000..36c7e8190 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,30 @@ +mod index; +mod server; +mod service; + +pub use server::Server; + +/// Performs a search test on both post and get routes +#[macro_export] +macro_rules! test_post_get_search { + ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => { + let post_query: meilisearch_http::routes::search::SearchQueryPost = + serde_json::from_str(&$query.clone().to_string()).unwrap(); + let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into(); + let get_query = ::serde_url_params::to_string(&get_query).unwrap(); + let ($response, $status_code) = $server.search_get(&get_query).await; + let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { + panic!( + "panic in get route: {:?}", + e.downcast_ref::<&str>().unwrap() + ) + }); + let ($response, $status_code) = $server.search_post($query).await; + let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { + panic!( + "panic in post route: {:?}", + e.downcast_ref::<&str>().unwrap() + ) + }); + }; +} diff --git a/tests/common/server.rs b/tests/common/server.rs new file mode 100644 index 000000000..5982af973 --- /dev/null +++ b/tests/common/server.rs @@ -0,0 +1,537 @@ +use tempdir::TempDir; +use byte_unit::{Byte, ByteUnit}; + +use meilisearch_http::data::Data; +use meilisearch_http::option::{Opt, IndexerOpts}; + +use super::index::Index; +use super::service::Service; + +pub struct Server { + service: Service, +} + +impl Server { + pub async fn new() -> Self { + let tmp_dir = TempDir::new("meilisearch").unwrap(); + + let opt = Opt { + db_path: tmp_dir.path().join("db"), + dumps_dir: tmp_dir.path().join("dump"), + dump_batch_size: 16, + http_addr: "127.0.0.1:7700".to_owned(), + master_key: None, + env: "development".to_owned(), + no_analytics: true, + max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), + max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), + http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), + ssl_cert_path: None, + ssl_key_path: None, + ssl_auth_path: None, + ssl_ocsp_path: None, + ssl_require_auth: false, + ssl_resumption: false, + ssl_tickets: false, + import_snapshot: None, + ignore_missing_snapshot: false, + ignore_snapshot_if_db_exists: false, + snapshot_dir: ".".into(), + schedule_snapshot: false, + snapshot_interval_sec: None, + import_dump: None, + indexer_options: IndexerOpts::default(), + }; + + let data = Data::new(opt).unwrap(); + let service = Service(data); + + Server { + service, + } + } + + //pub async fn test_server() -> Self { + //let mut server = Self::new(); + + //let body = json!({ + //"uid": "test", + //"primaryKey": "id", + //}); + + //server.create_index(body).await; + + //let body = json!({ + ////"rankingRules": [ + ////"typo", + ////"words", + ////"proximity", + ////"attribute", + ////"wordsPosition", + ////"exactness", + ////], + //"searchableAttributes": [ + //"balance", + //"picture", + //"age", + //"color", + //"name", + //"gender", + //"email", + //"phone", + //"address", + //"about", + //"registered", + //"latitude", + //"longitude", + //"tags", + //], + //"displayedAttributes": [ + //"id", + //"isActive", + //"balance", + //"picture", + //"age", + //"color", + //"name", + //"gender", + //"email", + //"phone", + //"address", + //"about", + //"registered", + //"latitude", + //"longitude", + //"tags", + //], + //}); + + //server.update_all_settings(body).await; + + //let dataset = include_bytes!("../assets/test_set.json"); + + //let body: Value = serde_json::from_slice(dataset).unwrap(); + + //server.add_or_replace_multiple_documents(body).await; + //server + //} + + //pub fn data(&self) -> &Data { + //&self.data + //} + + //pub async fn wait_update_id(&mut self, update_id: u64) { + //// try 10 times to get status, or panic to not wait forever + //for _ in 0..10 { + //let (response, status_code) = self.get_update_status(update_id).await; + //assert_eq!(status_code, 200); + + //if response["status"] == "processed" || response["status"] == "failed" { + //// eprintln!("{:#?}", response); + //return; + //} + + //delay_for(Duration::from_secs(1)).await; + //} + //panic!("Timeout waiting for update id"); + //} + + // Global Http request GET/POST/DELETE async or sync + + //pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) { + //eprintln!("get_request: {}", url); + + //let mut app = + //test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; + + //let req = test::TestRequest::get().uri(url).to_request(); + //let res = test::call_service(&mut app, req).await; + //let status_code = res.status(); + + //let body = test::read_body(res).await; + //let response = serde_json::from_slice(&body).unwrap_or_default(); + //(response, status_code) + //} + + + //pub async fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { + //eprintln!("post_request_async: {}", url); + + //let (response, status_code) = self.post_request(url, body).await; + //eprintln!("response: {}", response); + //assert!(response["updateId"].as_u64().is_some()); + //self.wait_update_id(response["updateId"].as_u64().unwrap()) + //.await; + //(response, status_code) + //} + + //pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { + //eprintln!("put_request: {}", url); + + //let mut app = + //test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; + + //let req = test::TestRequest::put() + //.uri(url) + //.set_json(&body) + //.to_request(); + //let res = test::call_service(&mut app, req).await; + //let status_code = res.status(); + + //let body = test::read_body(res).await; + //let response = serde_json::from_slice(&body).unwrap_or_default(); + //(response, status_code) + //} + + //pub async fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { + //eprintln!("put_request_async: {}", url); + + //let (response, status_code) = self.put_request(url, body).await; + //assert!(response["updateId"].as_u64().is_some()); + //assert_eq!(status_code, 202); + //self.wait_update_id(response["updateId"].as_u64().unwrap()) + //.await; + //(response, status_code) + //} + + //pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) { + //eprintln!("delete_request: {}", url); + + //let mut app = + //test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; + + //let req = test::TestRequest::delete().uri(url).to_request(); + //let res = test::call_service(&mut app, req).await; + //let status_code = res.status(); + + //let body = test::read_body(res).await; + //let response = serde_json::from_slice(&body).unwrap_or_default(); + //(response, status_code) + //} + + //pub async fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) { + //eprintln!("delete_request_async: {}", url); + + //let (response, status_code) = self.delete_request(url).await; + //assert!(response["updateId"].as_u64().is_some()); + //assert_eq!(status_code, 202); + //self.wait_update_id(response["updateId"].as_u64().unwrap()) + //.await; + //(response, status_code) + //} + + // All Routes + + //pub async fn list_indexes(&mut self) -> (Value, StatusCode) { + //self.get_request("/indexes").await + //} + + /// Returns a view to an index. There is no guarantee that the index exists. + pub fn index<'a>(&'a self, uid: impl AsRef) -> Index<'a> { + Index { + uid: uid.as_ref().to_string(), + service: &self.service, + } + } + //pub async fn search_multi_index(&mut self, query: &str) -> (Value, StatusCode) { + //let url = format!("/indexes/search?{}", query); + //self.get_request(&url).await + //} + + //pub async fn get_index(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_index(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}", self.uid); + //self.put_request(&url, body).await + //} + + //pub async fn delete_index(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}", self.uid); + //self.delete_request(&url).await + //} + + //pub async fn search_get(&mut self, query: &str) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/search?{}", self.uid, query); + //self.get_request(&url).await + //} + + //pub async fn search_post(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/search", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn get_all_updates_status(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/updates", self.uid); + //self.get_request(&url).await + //} + + //pub async fn get_update_status(&mut self, update_id: u64) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/updates/{}", self.uid, update_id); + //self.get_request(&url).await + //} + + //pub async fn get_all_documents(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/documents", self.uid); + //self.get_request(&url).await + //} + + //pub async fn add_or_replace_multiple_documents(&mut self, body: Value) { + //let url = format!("/indexes/{}/documents", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn add_or_replace_multiple_documents_sync( + //&mut self, + //body: Value, + //) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/documents", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn add_or_update_multiple_documents(&mut self, body: Value) { + //let url = format!("/indexes/{}/documents", self.uid); + //self.put_request_async(&url, body).await; + //} + + //pub async fn clear_all_documents(&mut self) { + //let url = format!("/indexes/{}/documents", self.uid); + //self.delete_request_async(&url).await; + //} + + //pub async fn get_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { + //let url = format!( + //"/indexes/{}/documents/{}", + //self.uid, + //document_id.to_string() + //); + //self.get_request(&url).await + //} + + //pub async fn delete_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { + //let url = format!( + //"/indexes/{}/documents/{}", + //self.uid, + //document_id.to_string() + //); + //self.delete_request_async(&url).await + //} + + //pub async fn delete_multiple_documents(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/documents/delete-batch", self.uid); + //self.post_request_async(&url, body).await + //} + + //pub async fn get_all_settings(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_all_settings(&mut self, body: Value) { + //let url = format!("/indexes/{}/settings", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn update_all_settings_sync(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn delete_all_settings(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings", self.uid); + //self.delete_request_async(&url).await + //} + + //pub async fn get_ranking_rules(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/ranking-rules", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_ranking_rules(&mut self, body: Value) { + //let url = format!("/indexes/{}/settings/ranking-rules", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn update_ranking_rules_sync(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/ranking-rules", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn delete_ranking_rules(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/ranking-rules", self.uid); + //self.delete_request_async(&url).await + //} + + //pub async fn get_distinct_attribute(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_distinct_attribute(&mut self, body: Value) { + //let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn update_distinct_attribute_sync(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); + //self.delete_request_async(&url).await + //} + + //pub async fn get_primary_key(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/primary_key", self.uid); + //self.get_request(&url).await + //} + + //pub async fn get_searchable_attributes(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_searchable_attributes(&mut self, body: Value) { + //let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn update_searchable_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); + //self.delete_request_async(&url).await + //} + + //pub async fn get_displayed_attributes(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_displayed_attributes(&mut self, body: Value) { + //let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn update_displayed_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); + //self.delete_request_async(&url).await + //} + + //pub async fn get_attributes_for_faceting(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_attributes_for_faceting(&mut self, body: Value) { + //let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn update_attributes_for_faceting_sync( + //&mut self, + //body: Value, + //) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn delete_attributes_for_faceting(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); + //self.delete_request_async(&url).await + //} + + //pub async fn get_synonyms(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/synonyms", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_synonyms(&mut self, body: Value) { + //let url = format!("/indexes/{}/settings/synonyms", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn update_synonyms_sync(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/synonyms", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn delete_synonyms(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/synonyms", self.uid); + //self.delete_request_async(&url).await + //} + + //pub async fn get_stop_words(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/stop-words", self.uid); + //self.get_request(&url).await + //} + + //pub async fn update_stop_words(&mut self, body: Value) { + //let url = format!("/indexes/{}/settings/stop-words", self.uid); + //self.post_request_async(&url, body).await; + //} + + //pub async fn update_stop_words_sync(&mut self, body: Value) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/stop-words", self.uid); + //self.post_request(&url, body).await + //} + + //pub async fn delete_stop_words(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/settings/stop-words", self.uid); + //self.delete_request_async(&url).await + //} + + //pub async fn get_index_stats(&mut self) -> (Value, StatusCode) { + //let url = format!("/indexes/{}/stats", self.uid); + //self.get_request(&url).await + //} + + //pub async fn list_keys(&mut self) -> (Value, StatusCode) { + //self.get_request("/keys").await + //} + + //pub async fn get_health(&mut self) -> (Value, StatusCode) { + //self.get_request("/health").await + //} + + //pub async fn update_health(&mut self, body: Value) -> (Value, StatusCode) { + //self.put_request("/health", body).await + //} + + //pub async fn get_version(&mut self) -> (Value, StatusCode) { + //self.get_request("/version").await + //} + + //pub async fn get_sys_info(&mut self) -> (Value, StatusCode) { + //self.get_request("/sys-info").await + //} + + //pub async fn get_sys_info_pretty(&mut self) -> (Value, StatusCode) { + //self.get_request("/sys-info/pretty").await + //} + + //pub async fn trigger_dump(&self) -> (Value, StatusCode) { + //self.post_request("/dumps", Value::Null).await + //} + + //pub async fn get_dump_status(&mut self, dump_uid: &str) -> (Value, StatusCode) { + //let url = format!("/dumps/{}/status", dump_uid); + //self.get_request(&url).await + //} + + //pub async fn trigger_dump_importation(&mut self, dump_uid: &str) -> (Value, StatusCode) { + //let url = format!("/dumps/{}/import", dump_uid); + //self.get_request(&url).await + //} +} diff --git a/tests/common/service.rs b/tests/common/service.rs new file mode 100644 index 000000000..21442cf64 --- /dev/null +++ b/tests/common/service.rs @@ -0,0 +1,39 @@ +use actix_web::{http::StatusCode, test}; +use serde_json::Value; + +use meilisearch_http::data::Data; +use meilisearch_http::helpers::NormalizePath; + +pub struct Service(pub Data); + +impl Service { + pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { + let mut app = + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::post() + .uri(url.as_ref()) + .set_json(&body) + .to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { + let mut app = + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::get().uri(url.as_ref()).to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } +} + diff --git a/tests/index/create_index.rs b/tests/index/create_index.rs new file mode 100644 index 000000000..b7bce4e26 --- /dev/null +++ b/tests/index/create_index.rs @@ -0,0 +1,48 @@ +use crate::common::Server; +use serde_json::Value; + +#[actix_rt::test] +async fn create_index_no_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + let (response, code) = index.create(None).await; + + assert_eq!(code, 200); + assert_eq!(response["uid"], "test"); + assert!(response.get("uuid").is_some()); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + assert_eq!(response["createdAt"], response["updatedAt"]); + assert_eq!(response["primaryKey"], Value::Null); + assert_eq!(response.as_object().unwrap().len(), 5); +} + +#[actix_rt::test] +async fn create_index_with_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + let (response, code) = index.create(Some("primary")).await; + + assert_eq!(code, 200); + assert_eq!(response["uid"], "test"); + assert!(response.get("uuid").is_some()); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + assert_eq!(response["createdAt"], response["updatedAt"]); + assert_eq!(response["primaryKey"], "primary"); + assert_eq!(response.as_object().unwrap().len(), 5); +} + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn create_existing_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(Some("primary")).await; + + assert_eq!(code, 200); + + let (_response, code) = index.create(Some("primary")).await; + assert_eq!(code, 400); +} diff --git a/tests/index/mod.rs b/tests/index/mod.rs new file mode 100644 index 000000000..cd3203e6b --- /dev/null +++ b/tests/index/mod.rs @@ -0,0 +1,2 @@ +mod create_index; +mod get_index; diff --git a/tests/integration.rs b/tests/integration.rs new file mode 100644 index 000000000..2b8dc5612 --- /dev/null +++ b/tests/integration.rs @@ -0,0 +1,3 @@ +mod common; +mod search; +mod index; diff --git a/tests/search/mod.rs b/tests/search/mod.rs new file mode 100644 index 000000000..735a3478f --- /dev/null +++ b/tests/search/mod.rs @@ -0,0 +1,3 @@ +// This modules contains all the test concerning search. Each particular feture of the search +// should be tested in its own module to isolate tests and keep the tests readable. + From 68692a256e726004a647927f66464fde8666fd35 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Feb 2021 19:52:00 +0100 Subject: [PATCH 061/522] test get index --- tests/index/create_index.rs | 10 ++++++++++ tests/index/get_index.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/index/get_index.rs diff --git a/tests/index/create_index.rs b/tests/index/create_index.rs index b7bce4e26..c118d5566 100644 --- a/tests/index/create_index.rs +++ b/tests/index/create_index.rs @@ -46,3 +46,13 @@ async fn create_existing_index() { let (_response, code) = index.create(Some("primary")).await; assert_eq!(code, 400); } + +// test fails (issue #46) +#[actix_rt::test] +#[ignore] +async fn create_with_invalid_index_uid() { + let server = Server::new().await; + let index = server.index("test test"); + let (_, code) = index.create(None).await; + assert_eq!(code, 400); +} diff --git a/tests/index/get_index.rs b/tests/index/get_index.rs new file mode 100644 index 000000000..8b0f27441 --- /dev/null +++ b/tests/index/get_index.rs @@ -0,0 +1,34 @@ +use crate::common::Server; +use serde_json::Value; + +#[actix_rt::test] +async fn create_and_get_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(None).await; + + assert_eq!(code, 200); + + let (response, code) = index.get().await; + + assert_eq!(code, 200); + assert_eq!(response["uid"], "test"); + assert!(response.get("uuid").is_some()); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + assert_eq!(response["createdAt"], response["updatedAt"]); + assert_eq!(response["primaryKey"], Value::Null); + assert_eq!(response.as_object().unwrap().len(), 5); +} + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn get_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + + let (_response, code) = index.get().await; + + assert_eq!(code, 400); +} From 4c5effe714764ac4be1e240157f35cb150259009 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Feb 2021 20:28:10 +0100 Subject: [PATCH 062/522] test index update --- src/data/updates.rs | 6 +++--- tests/common/index.rs | 9 +++++++++ tests/common/server.rs | 17 ----------------- tests/common/service.rs | 16 ++++++++++++++++ tests/index/mod.rs | 1 + 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 92d682b1d..fbb9be801 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,13 +1,13 @@ use std::ops::Deref; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; use async_compression::tokio_02::write::GzipEncoder; use futures_util::stream::StreamExt; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; use tokio::io::AsyncWriteExt; -use super::Data; -use crate::index_controller::{IndexController, Settings, IndexSettings, IndexMetadata}; use crate::index_controller::UpdateStatus; +use crate::index_controller::{IndexController, Settings, IndexSettings, IndexMetadata}; +use super::Data; impl Data { pub async fn add_documents( diff --git a/tests/common/index.rs b/tests/common/index.rs index 69b383b4a..5f297bbb5 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -24,4 +24,13 @@ impl Index<'_> { }); self.service.post("/indexes", body).await } + + pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) { + let body = json!({ + "primaryKey": primary_key, + }); + let url = format!("/indexes/{}", self.uid); + + self.service.put(url, body).await + } } diff --git a/tests/common/server.rs b/tests/common/server.rs index 5982af973..23aba5010 100644 --- a/tests/common/server.rs +++ b/tests/common/server.rs @@ -165,23 +165,6 @@ impl Server { //(response, status_code) //} - //pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - //eprintln!("put_request: {}", url); - - //let mut app = - //test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - //let req = test::TestRequest::put() - //.uri(url) - //.set_json(&body) - //.to_request(); - //let res = test::call_service(&mut app, req).await; - //let status_code = res.status(); - - //let body = test::read_body(res).await; - //let response = serde_json::from_slice(&body).unwrap_or_default(); - //(response, status_code) - //} //pub async fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { //eprintln!("put_request_async: {}", url); diff --git a/tests/common/service.rs b/tests/common/service.rs index 21442cf64..0ff1320df 100644 --- a/tests/common/service.rs +++ b/tests/common/service.rs @@ -35,5 +35,21 @@ impl Service { let response = serde_json::from_slice(&body).unwrap_or_default(); (response, status_code) } + + pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { + let mut app = + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::put() + .uri(url.as_ref()) + .set_json(&body) + .to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } } diff --git a/tests/index/mod.rs b/tests/index/mod.rs index cd3203e6b..2aefd99c2 100644 --- a/tests/index/mod.rs +++ b/tests/index/mod.rs @@ -1,2 +1,3 @@ mod create_index; mod get_index; +mod update_index; From ed3f8f5cc027ade2bc51cd6a3873a61423eff3ae Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Feb 2021 20:32:34 +0100 Subject: [PATCH 063/522] test create multiple indexes --- tests/index/create_index.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/index/create_index.rs b/tests/index/create_index.rs index c118d5566..3ff452c33 100644 --- a/tests/index/create_index.rs +++ b/tests/index/create_index.rs @@ -56,3 +56,21 @@ async fn create_with_invalid_index_uid() { let (_, code) = index.create(None).await; assert_eq!(code, 400); } + +#[actix_rt::test] +async fn test_create_multiple_indexes() { + let server = Server::new().await; + let index1 = server.index("test1"); + let index2 = server.index("test2"); + let index3 = server.index("test3"); + let index4 = server.index("test4"); + + index1.create(None).await; + index2.create(None).await; + index3.create(None).await; + + assert_eq!(index1.get().await.1, 200); + assert_eq!(index2.get().await.1, 200); + assert_eq!(index3.get().await.1, 200); + assert_eq!(index4.get().await.1, 400); +} From b293948d36f99bdd879d67e975cbc7cb67e522d7 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Feb 2021 20:44:33 +0100 Subject: [PATCH 064/522] test index delete --- tests/common/index.rs | 5 +++++ tests/common/server.rs | 30 ------------------------------ tests/common/service.rs | 13 +++++++++++++ tests/index/mod.rs | 1 + tests/integration.rs | 6 ++++++ 5 files changed, 25 insertions(+), 30 deletions(-) diff --git a/tests/common/index.rs b/tests/common/index.rs index 5f297bbb5..0b80f3fde 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -33,4 +33,9 @@ impl Index<'_> { self.service.put(url, body).await } + + pub async fn delete(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}", self.uid); + self.service.delete(url).await + } } diff --git a/tests/common/server.rs b/tests/common/server.rs index 23aba5010..30d6c40fd 100644 --- a/tests/common/server.rs +++ b/tests/common/server.rs @@ -138,22 +138,6 @@ impl Server { // Global Http request GET/POST/DELETE async or sync - //pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) { - //eprintln!("get_request: {}", url); - - //let mut app = - //test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - //let req = test::TestRequest::get().uri(url).to_request(); - //let res = test::call_service(&mut app, req).await; - //let status_code = res.status(); - - //let body = test::read_body(res).await; - //let response = serde_json::from_slice(&body).unwrap_or_default(); - //(response, status_code) - //} - - //pub async fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { //eprintln!("post_request_async: {}", url); @@ -177,20 +161,6 @@ impl Server { //(response, status_code) //} - //pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) { - //eprintln!("delete_request: {}", url); - - //let mut app = - //test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - //let req = test::TestRequest::delete().uri(url).to_request(); - //let res = test::call_service(&mut app, req).await; - //let status_code = res.status(); - - //let body = test::read_body(res).await; - //let response = serde_json::from_slice(&body).unwrap_or_default(); - //(response, status_code) - //} //pub async fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) { //eprintln!("delete_request_async: {}", url); diff --git a/tests/common/service.rs b/tests/common/service.rs index 0ff1320df..b00552562 100644 --- a/tests/common/service.rs +++ b/tests/common/service.rs @@ -51,5 +51,18 @@ impl Service { let response = serde_json::from_slice(&body).unwrap_or_default(); (response, status_code) } + + pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { + let mut app = + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } } diff --git a/tests/index/mod.rs b/tests/index/mod.rs index 2aefd99c2..253afcfc4 100644 --- a/tests/index/mod.rs +++ b/tests/index/mod.rs @@ -1,3 +1,4 @@ mod create_index; mod get_index; mod update_index; +mod delete_index; diff --git a/tests/integration.rs b/tests/integration.rs index 2b8dc5612..9d9ba64e3 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,3 +1,9 @@ mod common; mod search; mod index; + +// Tests are isolated by features in different modules to allow better readability, test +// targetability, and improved incremental compilation times. +// +// All the integration tests live in the same root module so only one test executable is generated, +// thus improving linking time. From b1226be2c870d11b61290f34bd0939c65c73d6ed Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Feb 2021 22:10:50 +0100 Subject: [PATCH 065/522] test document addition --- src/index_controller/updates.rs | 4 +++ tests/common/index.rs | 47 ++++++++++++++++++++++++++++++--- tests/common/service.rs | 1 + tests/integration.rs | 4 ++- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs index b8f1e31f4..b15593b58 100644 --- a/src/index_controller/updates.rs +++ b/src/index_controller/updates.rs @@ -42,6 +42,7 @@ impl Pending { } #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] pub struct Processed { pub success: N, pub processed_at: DateTime, @@ -56,6 +57,7 @@ impl Processed { } #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] pub struct Processing { #[serde(flatten)] pub from: Pending, @@ -89,6 +91,7 @@ impl Processing { } #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] pub struct Aborted { #[serde(flatten)] from: Pending, @@ -102,6 +105,7 @@ impl Aborted { } #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] pub struct Failed { #[serde(flatten)] from: Processing, diff --git a/tests/common/index.rs b/tests/common/index.rs index 0b80f3fde..182f9e238 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -1,5 +1,8 @@ +use std::time::Duration; + use actix_web::http::StatusCode; use serde_json::{json, Value}; +use tokio::time::delay_for; use super::service::Service; @@ -14,10 +17,7 @@ impl Index<'_> { self.service.get(url).await } - pub async fn create<'a>( - &'a self, - primary_key: Option<&str>, - ) -> (Value, StatusCode) { + pub async fn create<'a>(&'a self, primary_key: Option<&str>) -> (Value, StatusCode) { let body = json!({ "uid": self.uid, "primaryKey": primary_key, @@ -38,4 +38,43 @@ impl Index<'_> { let url = format!("/indexes/{}", self.uid); self.service.delete(url).await } + + pub async fn add_documents( + &self, + documents: Value, + primary_key: Option<&str>, + ) -> (Value, StatusCode) { + let url = match primary_key { + Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key), + None => format!("/indexes/{}/documents", self.uid), + }; + self.service.post(url, documents).await + } + + pub async fn wait_update_id(&self, update_id: u64) { + // try 10 times to get status, or panic to not wait forever + let url = format!("/indexes/{}/updates/{}", self.uid, update_id); + for _ in 0..10 { + let (response, status_code) = self.service.get(&url).await; + assert_eq!(status_code, 200); + + if response["status"] == "processed" || response["status"] == "failed" { + return; + } + + delay_for(Duration::from_secs(1)).await; + } + panic!("Timeout waiting for update id"); + } + + pub async fn get_update(&self, udpate_id: usize) -> (Value, StatusCode) { + let url = format!("/indexes/{}/updates/{}", self.uid, udpate_id); + self.service.get(url).await + } + + #[allow(dead_code)] + pub async fn list_updates(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/updates", self.uid); + self.service.get(url).await + } } diff --git a/tests/common/service.rs b/tests/common/service.rs index b00552562..8d19c4a49 100644 --- a/tests/common/service.rs +++ b/tests/common/service.rs @@ -64,5 +64,6 @@ impl Service { let response = serde_json::from_slice(&body).unwrap_or_default(); (response, status_code) } + } diff --git a/tests/integration.rs b/tests/integration.rs index 9d9ba64e3..34fbde4b6 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,6 +1,8 @@ mod common; -mod search; mod index; +mod search; +mod settings; +mod documents; // Tests are isolated by features in different modules to allow better readability, test // targetability, and improved incremental compilation times. From 556ba956b89de3069db89198d3ade95f509df625 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 19 Feb 2021 19:14:25 +0100 Subject: [PATCH 066/522] test get empty index list --- tests/common/server.rs | 433 +-------------------------------------- tests/index/get_index.rs | 9 + 2 files changed, 14 insertions(+), 428 deletions(-) diff --git a/tests/common/server.rs b/tests/common/server.rs index 30d6c40fd..630bb2791 100644 --- a/tests/common/server.rs +++ b/tests/common/server.rs @@ -1,5 +1,7 @@ use tempdir::TempDir; use byte_unit::{Byte, ByteUnit}; +use serde_json::Value; +use actix_web::http::StatusCode; use meilisearch_http::data::Data; use meilisearch_http::option::{Opt, IndexerOpts}; @@ -51,134 +53,6 @@ impl Server { } } - //pub async fn test_server() -> Self { - //let mut server = Self::new(); - - //let body = json!({ - //"uid": "test", - //"primaryKey": "id", - //}); - - //server.create_index(body).await; - - //let body = json!({ - ////"rankingRules": [ - ////"typo", - ////"words", - ////"proximity", - ////"attribute", - ////"wordsPosition", - ////"exactness", - ////], - //"searchableAttributes": [ - //"balance", - //"picture", - //"age", - //"color", - //"name", - //"gender", - //"email", - //"phone", - //"address", - //"about", - //"registered", - //"latitude", - //"longitude", - //"tags", - //], - //"displayedAttributes": [ - //"id", - //"isActive", - //"balance", - //"picture", - //"age", - //"color", - //"name", - //"gender", - //"email", - //"phone", - //"address", - //"about", - //"registered", - //"latitude", - //"longitude", - //"tags", - //], - //}); - - //server.update_all_settings(body).await; - - //let dataset = include_bytes!("../assets/test_set.json"); - - //let body: Value = serde_json::from_slice(dataset).unwrap(); - - //server.add_or_replace_multiple_documents(body).await; - //server - //} - - //pub fn data(&self) -> &Data { - //&self.data - //} - - //pub async fn wait_update_id(&mut self, update_id: u64) { - //// try 10 times to get status, or panic to not wait forever - //for _ in 0..10 { - //let (response, status_code) = self.get_update_status(update_id).await; - //assert_eq!(status_code, 200); - - //if response["status"] == "processed" || response["status"] == "failed" { - //// eprintln!("{:#?}", response); - //return; - //} - - //delay_for(Duration::from_secs(1)).await; - //} - //panic!("Timeout waiting for update id"); - //} - - // Global Http request GET/POST/DELETE async or sync - - //pub async fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - //eprintln!("post_request_async: {}", url); - - //let (response, status_code) = self.post_request(url, body).await; - //eprintln!("response: {}", response); - //assert!(response["updateId"].as_u64().is_some()); - //self.wait_update_id(response["updateId"].as_u64().unwrap()) - //.await; - //(response, status_code) - //} - - - //pub async fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - //eprintln!("put_request_async: {}", url); - - //let (response, status_code) = self.put_request(url, body).await; - //assert!(response["updateId"].as_u64().is_some()); - //assert_eq!(status_code, 202); - //self.wait_update_id(response["updateId"].as_u64().unwrap()) - //.await; - //(response, status_code) - //} - - - //pub async fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) { - //eprintln!("delete_request_async: {}", url); - - //let (response, status_code) = self.delete_request(url).await; - //assert!(response["updateId"].as_u64().is_some()); - //assert_eq!(status_code, 202); - //self.wait_update_id(response["updateId"].as_u64().unwrap()) - //.await; - //(response, status_code) - //} - - // All Routes - - //pub async fn list_indexes(&mut self) -> (Value, StatusCode) { - //self.get_request("/indexes").await - //} - /// Returns a view to an index. There is no guarantee that the index exists. pub fn index<'a>(&'a self, uid: impl AsRef) -> Index<'a> { Index { @@ -186,305 +60,8 @@ impl Server { service: &self.service, } } - //pub async fn search_multi_index(&mut self, query: &str) -> (Value, StatusCode) { - //let url = format!("/indexes/search?{}", query); - //self.get_request(&url).await - //} - //pub async fn get_index(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_index(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}", self.uid); - //self.put_request(&url, body).await - //} - - //pub async fn delete_index(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}", self.uid); - //self.delete_request(&url).await - //} - - //pub async fn search_get(&mut self, query: &str) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/search?{}", self.uid, query); - //self.get_request(&url).await - //} - - //pub async fn search_post(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/search", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn get_all_updates_status(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/updates", self.uid); - //self.get_request(&url).await - //} - - //pub async fn get_update_status(&mut self, update_id: u64) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/updates/{}", self.uid, update_id); - //self.get_request(&url).await - //} - - //pub async fn get_all_documents(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/documents", self.uid); - //self.get_request(&url).await - //} - - //pub async fn add_or_replace_multiple_documents(&mut self, body: Value) { - //let url = format!("/indexes/{}/documents", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn add_or_replace_multiple_documents_sync( - //&mut self, - //body: Value, - //) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/documents", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn add_or_update_multiple_documents(&mut self, body: Value) { - //let url = format!("/indexes/{}/documents", self.uid); - //self.put_request_async(&url, body).await; - //} - - //pub async fn clear_all_documents(&mut self) { - //let url = format!("/indexes/{}/documents", self.uid); - //self.delete_request_async(&url).await; - //} - - //pub async fn get_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { - //let url = format!( - //"/indexes/{}/documents/{}", - //self.uid, - //document_id.to_string() - //); - //self.get_request(&url).await - //} - - //pub async fn delete_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { - //let url = format!( - //"/indexes/{}/documents/{}", - //self.uid, - //document_id.to_string() - //); - //self.delete_request_async(&url).await - //} - - //pub async fn delete_multiple_documents(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/documents/delete-batch", self.uid); - //self.post_request_async(&url, body).await - //} - - //pub async fn get_all_settings(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_all_settings(&mut self, body: Value) { - //let url = format!("/indexes/{}/settings", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn update_all_settings_sync(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn delete_all_settings(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings", self.uid); - //self.delete_request_async(&url).await - //} - - //pub async fn get_ranking_rules(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_ranking_rules(&mut self, body: Value) { - //let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn update_ranking_rules_sync(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn delete_ranking_rules(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - //self.delete_request_async(&url).await - //} - - //pub async fn get_distinct_attribute(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_distinct_attribute(&mut self, body: Value) { - //let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn update_distinct_attribute_sync(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - //self.delete_request_async(&url).await - //} - - //pub async fn get_primary_key(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/primary_key", self.uid); - //self.get_request(&url).await - //} - - //pub async fn get_searchable_attributes(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_searchable_attributes(&mut self, body: Value) { - //let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn update_searchable_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - //self.delete_request_async(&url).await - //} - - //pub async fn get_displayed_attributes(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_displayed_attributes(&mut self, body: Value) { - //let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn update_displayed_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - //self.delete_request_async(&url).await - //} - - //pub async fn get_attributes_for_faceting(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_attributes_for_faceting(&mut self, body: Value) { - //let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn update_attributes_for_faceting_sync( - //&mut self, - //body: Value, - //) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn delete_attributes_for_faceting(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - //self.delete_request_async(&url).await - //} - - //pub async fn get_synonyms(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/synonyms", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_synonyms(&mut self, body: Value) { - //let url = format!("/indexes/{}/settings/synonyms", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn update_synonyms_sync(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/synonyms", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn delete_synonyms(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/synonyms", self.uid); - //self.delete_request_async(&url).await - //} - - //pub async fn get_stop_words(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/stop-words", self.uid); - //self.get_request(&url).await - //} - - //pub async fn update_stop_words(&mut self, body: Value) { - //let url = format!("/indexes/{}/settings/stop-words", self.uid); - //self.post_request_async(&url, body).await; - //} - - //pub async fn update_stop_words_sync(&mut self, body: Value) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/stop-words", self.uid); - //self.post_request(&url, body).await - //} - - //pub async fn delete_stop_words(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/settings/stop-words", self.uid); - //self.delete_request_async(&url).await - //} - - //pub async fn get_index_stats(&mut self) -> (Value, StatusCode) { - //let url = format!("/indexes/{}/stats", self.uid); - //self.get_request(&url).await - //} - - //pub async fn list_keys(&mut self) -> (Value, StatusCode) { - //self.get_request("/keys").await - //} - - //pub async fn get_health(&mut self) -> (Value, StatusCode) { - //self.get_request("/health").await - //} - - //pub async fn update_health(&mut self, body: Value) -> (Value, StatusCode) { - //self.put_request("/health", body).await - //} - - //pub async fn get_version(&mut self) -> (Value, StatusCode) { - //self.get_request("/version").await - //} - - //pub async fn get_sys_info(&mut self) -> (Value, StatusCode) { - //self.get_request("/sys-info").await - //} - - //pub async fn get_sys_info_pretty(&mut self) -> (Value, StatusCode) { - //self.get_request("/sys-info/pretty").await - //} - - //pub async fn trigger_dump(&self) -> (Value, StatusCode) { - //self.post_request("/dumps", Value::Null).await - //} - - //pub async fn get_dump_status(&mut self, dump_uid: &str) -> (Value, StatusCode) { - //let url = format!("/dumps/{}/status", dump_uid); - //self.get_request(&url).await - //} - - //pub async fn trigger_dump_importation(&mut self, dump_uid: &str) -> (Value, StatusCode) { - //let url = format!("/dumps/{}/import", dump_uid); - //self.get_request(&url).await - //} + pub async fn list_indexes(&self) -> (Value, StatusCode) { + self.service.get("/indexes").await + } } diff --git a/tests/index/get_index.rs b/tests/index/get_index.rs index 8b0f27441..c69f4b2a8 100644 --- a/tests/index/get_index.rs +++ b/tests/index/get_index.rs @@ -32,3 +32,12 @@ async fn get_unexisting_index() { assert_eq!(code, 400); } + +#[actix_rt::test] +async fn no_index_return_empty_list() { + let server = Server::new().await; + let (response, code) = server.list_indexes().await; + assert_eq!(code, 200); + assert!(response.is_array()); + assert!(response.as_array().unwrap().is_empty()); +} From 2bb695d60f55ddbc9095c208377caf8f60ed315b Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 19 Feb 2021 19:23:58 +0100 Subject: [PATCH 067/522] test list all indexes --- tests/index/get_index.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/index/get_index.rs b/tests/index/get_index.rs index c69f4b2a8..3b8390551 100644 --- a/tests/index/get_index.rs +++ b/tests/index/get_index.rs @@ -41,3 +41,19 @@ async fn no_index_return_empty_list() { assert!(response.is_array()); assert!(response.as_array().unwrap().is_empty()); } + +#[actix_rt::test] +async fn list_multiple_indexes() { + let server = Server::new().await; + server.index("test").create(None).await; + server.index("test1").create(Some("key")).await; + + let (response, code) = server.list_indexes().await; + assert_eq!(code, 200); + assert!(response.is_array()); + let arr = response.as_array().unwrap(); + assert_eq!(arr.len(), 2); + assert!(arr.iter().find(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null).is_some()); + assert!(arr.iter().find(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key").is_some()); + +} From 5270cc0eae1d4f57b8775668fbba3b3ce3afc408 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 19 Feb 2021 19:26:42 +0100 Subject: [PATCH 068/522] test update index --- tests/index/update_index.rs | 64 +++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/index/update_index.rs diff --git a/tests/index/update_index.rs b/tests/index/update_index.rs new file mode 100644 index 000000000..0078ad892 --- /dev/null +++ b/tests/index/update_index.rs @@ -0,0 +1,64 @@ +use crate::common::Server; +use chrono::DateTime; + +#[actix_rt::test] +async fn update_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(None).await; + + assert_eq!(code, 200); + + let (response, code) = index.update(Some("primary")).await; + + assert_eq!(code, 200); + assert_eq!(response["uid"], "test"); + assert!(response.get("uuid").is_some()); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + + let created_at = DateTime::parse_from_rfc3339(response["createdAt"].as_str().unwrap()).unwrap(); + let updated_at = DateTime::parse_from_rfc3339(response["updatedAt"].as_str().unwrap()).unwrap(); + assert!(created_at < updated_at); + + assert_eq!(response["primaryKey"], "primary"); + assert_eq!(response.as_object().unwrap().len(), 5); +} + +#[actix_rt::test] +async fn update_nothing() { + let server = Server::new().await; + let index = server.index("test"); + let (response, code) = index.create(None).await; + + assert_eq!(code, 200); + + let (update, code) = index.update(None).await; + + assert_eq!(code, 200); + assert_eq!(response, update); +} + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn update_existing_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.create(Some("primary")).await; + + assert_eq!(code, 200); + + let (_update, code) = index.update(Some("primary2")).await; + + assert_eq!(code, 400); +} + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn test_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").update(None).await; + assert_eq!(code, 400); +} From ba2cfcc72d3f56e7221daeae005905f8b4c3d4ec Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 19 Feb 2021 19:26:56 +0100 Subject: [PATCH 069/522] test delete index --- tests/index/delete_index.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/index/delete_index.rs diff --git a/tests/index/delete_index.rs b/tests/index/delete_index.rs new file mode 100644 index 000000000..39e79daaf --- /dev/null +++ b/tests/index/delete_index.rs @@ -0,0 +1,25 @@ +use crate::common::Server; + +#[actix_rt::test] +async fn create_and_delete_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.create(None).await; + + assert_eq!(code, 200); + + let (_response, code) = index.delete().await; + + assert_eq!(code, 200); + + assert_eq!(index.get().await.1, 400); +} + +#[actix_rt::test] +async fn delete_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.delete().await; + + assert_eq!(code, 400); +} From ec9dcd3285de83a7e562dac1c7ad0ab3770db8de Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 19 Feb 2021 19:43:32 +0100 Subject: [PATCH 070/522] test get add documents --- tests/documents/add_documents.rs | 210 +++++++++++++++++++++++++++++++ tests/documents/get_documents.rs | 24 ++++ tests/documents/mod.rs | 2 + 3 files changed, 236 insertions(+) create mode 100644 tests/documents/add_documents.rs create mode 100644 tests/documents/get_documents.rs create mode 100644 tests/documents/mod.rs diff --git a/tests/documents/add_documents.rs b/tests/documents/add_documents.rs new file mode 100644 index 000000000..c17828641 --- /dev/null +++ b/tests/documents/add_documents.rs @@ -0,0 +1,210 @@ +use serde_json::{json, Value}; +use chrono::DateTime; + +use crate::common::Server; + +#[actix_rt::test] +async fn add_documents_no_index_creation() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "id": 1, + "content": "foo", + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "pending"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["meta"]["type"], "DocumentsAddition"); + assert_eq!(response["meta"]["format"], "Json"); + assert_eq!(response["meta"]["primaryKey"], Value::Null); + assert!(response.get("enqueuedAt").is_some()); + + index.wait_update_id(0).await; + + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + println!("response: {}", response); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + + let processed_at = DateTime::parse_from_rfc3339(response["processedAt"].as_str().unwrap()).unwrap(); + let enqueued_at = DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap(); + let started_processing_at = DateTime::parse_from_rfc3339(response["startedProcessingAt"].as_str().unwrap()).unwrap(); + assert!(processed_at > started_processing_at); + assert!(started_processing_at > enqueued_at); + + // index was created, and primary key was infered. + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "id"); +} + +#[actix_rt::test] +async fn document_addition_with_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "primary": 1, + "content": "foo", + } + ]); + let (_response, code) = index.add_documents(documents, Some("primary")).await; + assert_eq!(code, 200); + + index.wait_update_id(0).await; + + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + +#[actix_rt::test] +async fn add_documents_with_primary_key_and_primary_key_already_exists() { + let server = Server::new().await; + let index = server.index("test"); + + index.create(Some("primary")).await; + let documents = json!([ + { + "id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.add_documents(documents, Some("id")).await; + assert_eq!(code, 200); + + index.wait_update_id(0).await; + + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + +#[actix_rt::test] +async fn replace_document() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "doc_id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 200); + + index.wait_update_id(0).await; + + let documents = json!([ + { + "doc_id": 1, + "other": "bar", + } + ]); + + let (_response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 200); + + index.wait_update_id(1).await; + + let (response, code) = index.get_update(1).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert_eq!(response.to_string(), r##"{"doc_id":1,"other":"bar"}"##); +} + +// test broken, see issue milli#92 +#[actix_rt::test] +#[ignore] +async fn add_no_documents() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.add_documents(json!([]), None).await; + assert_eq!(code, 200); + + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 0); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], Value::Null); +} + +#[actix_rt::test] +async fn update_document() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "doc_id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 200); + + index.wait_update_id(0).await; + + let documents = json!([ + { + "doc_id": 1, + "other": "bar", + } + ]); + + let (_response, code) = index.update_documents(documents, None).await; + assert_eq!(code, 200); + + index.wait_update_id(1).await; + + let (response, code) = index.get_update(1).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##); +} + +#[actix_rt::test] +async fn add_larger_dataset() { + let server = Server::new().await; + let index = server.index("test"); + let update_id = index.load_test_set().await; + let (response, code) = index.get_update(update_id).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); +} diff --git a/tests/documents/get_documents.rs b/tests/documents/get_documents.rs new file mode 100644 index 000000000..40748b555 --- /dev/null +++ b/tests/documents/get_documents.rs @@ -0,0 +1,24 @@ +use crate::common::Server; +use crate::common::GetAllDocumentsOptions; + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn get_unexisting_index_single_document() { + let server = Server::new().await; + let (_response, code) = server + .index("test") + .get_document(1, None) + .await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn get_unexisting_index_all_documents() { + let server = Server::new().await; + let (_response, code) = server + .index("test") + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 400); +} diff --git a/tests/documents/mod.rs b/tests/documents/mod.rs new file mode 100644 index 000000000..69c73e37a --- /dev/null +++ b/tests/documents/mod.rs @@ -0,0 +1,2 @@ +mod add_documents; +mod get_documents; From 27a7238d3fe8848de8123fc28fe3ce3df31b2811 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 19 Feb 2021 19:46:45 +0100 Subject: [PATCH 071/522] test list no documents --- tests/documents/get_documents.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/documents/get_documents.rs b/tests/documents/get_documents.rs index 40748b555..069d5fd90 100644 --- a/tests/documents/get_documents.rs +++ b/tests/documents/get_documents.rs @@ -22,3 +22,15 @@ async fn get_unexisting_index_all_documents() { .await; assert_eq!(code, 400); } + +#[actix_rt::test] +async fn get_no_documents() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(None).await; + assert_eq!(code, 200); + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + assert_eq!(code, 200); + assert!(response.as_array().unwrap().is_empty()); +} From b8b8cc13127240dcb8de942d4dae6da71fbb05c8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 19 Feb 2021 19:55:00 +0100 Subject: [PATCH 072/522] get all documents, no options --- tests/common/index.rs | 34 +++++++++++++++++++++++++++++++- tests/documents/get_documents.rs | 12 +++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/common/index.rs b/tests/common/index.rs index 182f9e238..6e3dc2603 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -17,6 +17,15 @@ impl Index<'_> { self.service.get(url).await } + pub async fn load_test_set(&self) -> u64 { + let url = format!("/indexes/{}/documents", self.uid); + let (response, code) = self.service.post_str(url, include_str!("../assets/test_set.json")).await; + assert_eq!(code, 200); + let update_id = response["updateId"].as_i64().unwrap(); + self.wait_update_id(update_id as u64).await; + update_id as u64 + } + pub async fn create<'a>(&'a self, primary_key: Option<&str>) -> (Value, StatusCode) { let body = json!({ "uid": self.uid, @@ -51,6 +60,14 @@ impl Index<'_> { self.service.post(url, documents).await } + pub async fn update_documents(&self, documents: Value, primary_key: Option<&str>) -> (Value, StatusCode) { + let url = match primary_key { + Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key), + None => format!("/indexes/{}/documents", self.uid), + }; + self.service.put(url, documents).await + } + pub async fn wait_update_id(&self, update_id: u64) { // try 10 times to get status, or panic to not wait forever let url = format!("/indexes/{}/updates/{}", self.uid, update_id); @@ -67,7 +84,7 @@ impl Index<'_> { panic!("Timeout waiting for update id"); } - pub async fn get_update(&self, udpate_id: usize) -> (Value, StatusCode) { + pub async fn get_update(&self, udpate_id: u64) -> (Value, StatusCode) { let url = format!("/indexes/{}/updates/{}", self.uid, udpate_id); self.service.get(url).await } @@ -77,4 +94,19 @@ impl Index<'_> { let url = format!("/indexes/{}/updates", self.uid); self.service.get(url).await } + + pub async fn get_document(&self, id: u64, _options: Option) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents/{}", self.uid, id); + self.service.get(url).await + } + + pub async fn get_all_documents(&self, _options: GetAllDocumentsOptions) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents", self.uid); + self.service.get(url).await + } } + +pub struct GetDocumentOptions; + +#[derive(Debug, Default)] +pub struct GetAllDocumentsOptions; diff --git a/tests/documents/get_documents.rs b/tests/documents/get_documents.rs index 069d5fd90..f5afc29e2 100644 --- a/tests/documents/get_documents.rs +++ b/tests/documents/get_documents.rs @@ -34,3 +34,15 @@ async fn get_no_documents() { assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } + +#[actix_rt::test] +async fn get_all_documents_no_options() { + let server = Server::new().await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + assert_eq!(code, 200); + let arr = response.as_array().unwrap(); + assert_eq!(arr.len(), 20); +} From 097cae90a7455b486808cb808f0be5d0f0157949 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Feb 2021 14:23:17 +0100 Subject: [PATCH 073/522] tests get documents limit, offset, attr to retrieve --- tests/common/index.rs | 22 +++++++++-- tests/common/mod.rs | 1 + tests/common/service.rs | 20 +++++++++- tests/documents/get_documents.rs | 66 ++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 5 deletions(-) diff --git a/tests/common/index.rs b/tests/common/index.rs index 6e3dc2603..ed5e533a5 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -100,8 +100,20 @@ impl Index<'_> { self.service.get(url).await } - pub async fn get_all_documents(&self, _options: GetAllDocumentsOptions) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", self.uid); + pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) { + let mut url = format!("/indexes/{}/documents?", self.uid); + if let Some(limit) = options.limit { + url.push_str(&format!("limit={}&", limit)); + } + + if let Some(offset) = options.offset { + url.push_str(&format!("offset={}&", offset)); + } + + if let Some(attributes_to_retrieve) = options.attributes_to_retrieve { + url.push_str(&format!("attributesToRetrieve={}&", attributes_to_retrieve.join(","))); + } + self.service.get(url).await } } @@ -109,4 +121,8 @@ impl Index<'_> { pub struct GetDocumentOptions; #[derive(Debug, Default)] -pub struct GetAllDocumentsOptions; +pub struct GetAllDocumentsOptions { + pub limit: Option, + pub offset: Option, + pub attributes_to_retrieve: Option>, +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 36c7e8190..fd461e61c 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -3,6 +3,7 @@ mod server; mod service; pub use server::Server; +pub use index::{GetDocumentOptions, GetAllDocumentsOptions}; /// Performs a search test on both post and get routes #[macro_export] diff --git a/tests/common/service.rs b/tests/common/service.rs index 8d19c4a49..8e797c00d 100644 --- a/tests/common/service.rs +++ b/tests/common/service.rs @@ -23,6 +23,24 @@ impl Service { (response, status_code) } + /// Send a test post request from a text body, with a `content-type:application/json` header. + pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { + let mut app = + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + + let req = test::TestRequest::post() + .uri(url.as_ref()) + .set_payload(body.as_ref().to_string()) + .header("content-type", "application/json") + .to_request(); + let res = test::call_service(&mut app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; @@ -64,6 +82,4 @@ impl Service { let response = serde_json::from_slice(&body).unwrap_or_default(); (response, status_code) } - } - diff --git a/tests/documents/get_documents.rs b/tests/documents/get_documents.rs index f5afc29e2..3b4581096 100644 --- a/tests/documents/get_documents.rs +++ b/tests/documents/get_documents.rs @@ -45,4 +45,70 @@ async fn get_all_documents_no_options() { assert_eq!(code, 200); let arr = response.as_array().unwrap(); assert_eq!(arr.len(), 20); + let first = serde_json::json!({ + "id":0, + "isActive":false, + "balance":"$2,668.55", + "picture":"http://placehold.it/32x32", + "age":36, + "color":"Green", + "name":"Lucas Hess", + "gender":"male", + "email":"lucashess@chorizon.com", + "phone":"+1 (998) 478-2597", + "address":"412 Losee Terrace, Blairstown, Georgia, 2825", + "about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered":"2016-06-21T09:30:25 -02:00", + "latitude":-44.174957, + "longitude":-145.725388, + "tags":["bug" + ,"bug"]}); + assert_eq!(first, arr[0]); +} + +#[actix_rt::test] +async fn test_get_all_documents_limit() { + let server = Server::new().await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(5), ..Default::default() }).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 5); + assert_eq!(response.as_array().unwrap()[0]["id"], 0); +} + +#[actix_rt::test] +async fn test_get_all_documents_offset() { + let server = Server::new().await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions { offset: Some(5), ..Default::default() }).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response.as_array().unwrap()[0]["id"], 13); +} + +#[actix_rt::test] +async fn test_get_all_documents_attributes_to_retrieve() { + let server = Server::new().await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec!["name"]), ..Default::default() }).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 1); + assert!(response.as_array().unwrap()[0].as_object().unwrap().get("name").is_some()); + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec![]), ..Default::default() }).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 0); + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec!["name", "tags"]), ..Default::default() }).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 2); } From ded6483173ff052589831f44d39497dacfcbcf75 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Feb 2021 14:32:48 +0100 Subject: [PATCH 074/522] tests get one document --- tests/documents/get_documents.rs | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/documents/get_documents.rs b/tests/documents/get_documents.rs index 3b4581096..5affb8a7a 100644 --- a/tests/documents/get_documents.rs +++ b/tests/documents/get_documents.rs @@ -13,6 +13,41 @@ async fn get_unexisting_index_single_document() { assert_eq!(code, 400); } +#[actix_rt::test] +async fn get_unexisting_document() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (_response, code) = index + .get_document(1, None) + .await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn get_document() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let documents = serde_json::json!([ + { + "id": 0, + "content": "foobar", + } + ]); + let (_, code) = index.add_documents(documents, None).await; + assert_eq!(code, 200); + index.wait_update_id(0).await; + let (response, code) = index + .get_document(0, None) + .await; + assert_eq!(code, 200); + assert_eq!(response, serde_json::json!( { + "id": 0, + "content": "foobar", + })); +} + #[actix_rt::test] async fn get_unexisting_index_all_documents() { let server = Server::new().await; From 4bca26298ef615594c87049b75286ada871412af Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Feb 2021 14:55:40 +0100 Subject: [PATCH 075/522] test add document bad primary key --- tests/documents/add_documents.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/documents/add_documents.rs b/tests/documents/add_documents.rs index c17828641..568a3117e 100644 --- a/tests/documents/add_documents.rs +++ b/tests/documents/add_documents.rs @@ -208,3 +208,21 @@ async fn add_larger_dataset() { assert_eq!(response["status"], "processed"); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); } + +#[actix_rt::test] +async fn add_documents_bad_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + index.create(Some("docid")).await; + let documents = json!([ + { + "docid": "foo & bar", + "content": "foobar" + } + ]); + index.add_documents(documents, None).await; + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "failed"); +} From c95bf0cdf02c46bf54c0e468095e553a817a74c8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Feb 2021 15:13:10 +0100 Subject: [PATCH 076/522] test badly formated primary key --- tests/documents/add_documents.rs | 94 +++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/tests/documents/add_documents.rs b/tests/documents/add_documents.rs index 568a3117e..70d6aab68 100644 --- a/tests/documents/add_documents.rs +++ b/tests/documents/add_documents.rs @@ -1,7 +1,7 @@ use serde_json::{json, Value}; use chrono::DateTime; -use crate::common::Server; +use crate::common::{Server, GetAllDocumentsOptions}; #[actix_rt::test] async fn add_documents_no_index_creation() { @@ -56,8 +56,33 @@ async fn document_addition_with_primary_key() { "content": "foo", } ]); - let (_response, code) = index.add_documents(documents, Some("primary")).await; + let (_response, code) = index.add_documents(documents, Some("primary")).await; assert_eq!(code, 200); + + index.wait_update_id(0).await; + + let (response, code) = index.get_update(0).await; assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + +#[actix_rt::test] +async fn document_update_with_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "primary": 1, + "content": "foo", + } + ]); + let (_response, code) = index.update_documents(documents, Some("primary")).await; assert_eq!(code, 200); index.wait_update_id(0).await; @@ -101,6 +126,34 @@ async fn add_documents_with_primary_key_and_primary_key_already_exists() { assert_eq!(response["primaryKey"], "primary"); } +#[actix_rt::test] +async fn update_documents_with_primary_key_and_primary_key_already_exists() { + let server = Server::new().await; + let index = server.index("test"); + + index.create(Some("primary")).await; + let documents = json!([ + { + "id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.update_documents(documents, Some("id")).await; + assert_eq!(code, 200); + + index.wait_update_id(0).await; +let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + #[actix_rt::test] async fn replace_document() { let server = Server::new().await; @@ -207,6 +260,25 @@ async fn add_larger_dataset() { assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); + let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(1000), ..Default::default() }).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 77); +} + +#[actix_rt::test] +async fn update_larger_dataset() { + let server = Server::new().await; + let index = server.index("test"); + let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(); + index.update_documents(documents, None).await; + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); + let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(1000), ..Default::default() }).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 77); } #[actix_rt::test] @@ -226,3 +298,21 @@ async fn add_documents_bad_primary_key() { assert_eq!(code, 200); assert_eq!(response["status"], "failed"); } + +#[actix_rt::test] +async fn update_documents_bad_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + index.create(Some("docid")).await; + let documents = json!([ + { + "docid": "foo & bar", + "content": "foobar" + } + ]); + index.update_documents(documents, None).await; + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "failed"); +} From d3758b6f76d16dd8d0a506edf8a5a6a4a0949bc6 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Feb 2021 16:03:17 +0100 Subject: [PATCH 077/522] test delete documents --- tests/common/index.rs | 14 +++++- tests/documents/delete_documents.rs | 72 +++++++++++++++++++++++++++++ tests/documents/mod.rs | 1 + 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 tests/documents/delete_documents.rs diff --git a/tests/common/index.rs b/tests/common/index.rs index ed5e533a5..8afe78a35 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -68,7 +68,7 @@ impl Index<'_> { self.service.put(url, documents).await } - pub async fn wait_update_id(&self, update_id: u64) { + pub async fn wait_update_id(&self, update_id: u64) -> Value { // try 10 times to get status, or panic to not wait forever let url = format!("/indexes/{}/updates/{}", self.uid, update_id); for _ in 0..10 { @@ -76,7 +76,7 @@ impl Index<'_> { assert_eq!(status_code, 200); if response["status"] == "processed" || response["status"] == "failed" { - return; + return response; } delay_for(Duration::from_secs(1)).await; @@ -116,6 +116,16 @@ impl Index<'_> { self.service.get(url).await } + + pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents/{}", self.uid, id); + self.service.delete(url).await + } + + pub async fn clear_all_documents(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents", self.uid); + self.service.delete(url).await + } } pub struct GetDocumentOptions; diff --git a/tests/documents/delete_documents.rs b/tests/documents/delete_documents.rs new file mode 100644 index 000000000..149319308 --- /dev/null +++ b/tests/documents/delete_documents.rs @@ -0,0 +1,72 @@ +use serde_json::json; + +use crate::common::{Server, GetAllDocumentsOptions}; + +#[actix_rt::test] +async fn delete_one_document_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").delete_document(0).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn delete_one_unexisting_document() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (_response, code) = index.delete_document(0).await; + assert_eq!(code, 200); + let update = index.wait_update_id(0).await; + assert_eq!(update["status"], "processed"); +} + +#[actix_rt::test] +async fn delete_one_document() { + let server = Server::new().await; + let index = server.index("test"); + index.add_documents(json!([{ "id": 0, "content": "foobar" }]), None).await; + index.wait_update_id(0).await; + let (_response, code) = server.index("test").delete_document(0).await; + assert_eq!(code, 200); + + let (_response, code) = index.get_document(0, None).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn clear_all_documents_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").clear_all_documents().await; + assert_eq!(code, 400); + +} + +#[actix_rt::test] +async fn clear_all_documents() { + let server = Server::new().await; + let index = server.index("test"); + index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }]), None).await; + index.wait_update_id(0).await; + let (_response, code) = index.clear_all_documents().await; + assert_eq!(code, 200); + + let _update = index.wait_update_id(0).await; + let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + assert_eq!(code, 200); + assert!(response.as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn clear_all_documents_empty_index() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + + let (_response, code) = index.clear_all_documents().await; + assert_eq!(code, 200); + + let _update = index.wait_update_id(0).await; + let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + assert_eq!(code, 200); + assert!(response.as_array().unwrap().is_empty()); +} diff --git a/tests/documents/mod.rs b/tests/documents/mod.rs index 69c73e37a..ea0e39c69 100644 --- a/tests/documents/mod.rs +++ b/tests/documents/mod.rs @@ -1,2 +1,3 @@ mod add_documents; mod get_documents; +mod delete_documents; From 0a3e946726e0aa11b05c7399ddacb421ef1bd020 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Feb 2021 14:13:43 +0100 Subject: [PATCH 078/522] test delete batches --- src/option.rs | 2 +- tests/common/index.rs | 5 ++++ tests/common/server.rs | 4 +++ tests/documents/delete_documents.rs | 42 ++++++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/option.rs b/src/option.rs index c88110ccd..0a1d494de 100644 --- a/src/option.rs +++ b/src/option.rs @@ -103,8 +103,8 @@ pub struct Opt { pub sentry_dsn: String, /// Disable Sentry error reporting. - #[cfg(all(not(debug_assertions), feature = "sentry"))] #[structopt(long, env = "MEILI_NO_SENTRY")] + #[cfg(all(not(debug_assertions), feature = "sentry"))] pub no_sentry: bool, /// This environment variable must be set to `production` if you are running in production. diff --git a/tests/common/index.rs b/tests/common/index.rs index 8afe78a35..a50f8e6a3 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -126,6 +126,11 @@ impl Index<'_> { let url = format!("/indexes/{}/documents", self.uid); self.service.delete(url).await } + + pub async fn delete_batch(&self, ids: Vec) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents/delete-batch", self.uid); + self.service.post(url, serde_json::to_value(&ids).unwrap()).await + } } pub struct GetDocumentOptions; diff --git a/tests/common/server.rs b/tests/common/server.rs index 630bb2791..de32a2045 100644 --- a/tests/common/server.rs +++ b/tests/common/server.rs @@ -43,6 +43,10 @@ impl Server { snapshot_interval_sec: None, import_dump: None, indexer_options: IndexerOpts::default(), + #[cfg(all(not(debug_assertions), feature = "sentry"))] + sentry_dsn: String::from(""), + #[cfg(all(not(debug_assertions), feature = "sentry"))] + no_sentry: true, }; let data = Data::new(opt).unwrap(); diff --git a/tests/documents/delete_documents.rs b/tests/documents/delete_documents.rs index 149319308..3dba7e6fd 100644 --- a/tests/documents/delete_documents.rs +++ b/tests/documents/delete_documents.rs @@ -28,6 +28,7 @@ async fn delete_one_document() { index.wait_update_id(0).await; let (_response, code) = server.index("test").delete_document(0).await; assert_eq!(code, 200); + index.wait_update_id(1).await; let (_response, code) = index.get_document(0, None).await; assert_eq!(code, 400); @@ -50,7 +51,7 @@ async fn clear_all_documents() { let (_response, code) = index.clear_all_documents().await; assert_eq!(code, 200); - let _update = index.wait_update_id(0).await; + let _update = index.wait_update_id(1).await; let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); @@ -70,3 +71,42 @@ async fn clear_all_documents_empty_index() { assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } + +#[actix_rt::test] +async fn delete_batch_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").delete_batch(vec![]).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn delete_batch() { + let server = Server::new().await; + let index = server.index("test"); + index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; + index.wait_update_id(0).await; + let (_response, code) = index.delete_batch(vec![1, 0]).await; + assert_eq!(code, 200); + + let _update = index.wait_update_id(1).await; + let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 1); + assert_eq!(response.as_array().unwrap()[0]["id"], 3); +} + + +#[actix_rt::test] +async fn delete_no_document_batch() { + let server = Server::new().await; + let index = server.index("test"); + index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; + index.wait_update_id(0).await; + let (_response, code) = index.delete_batch(vec![]).await; + assert_eq!(code, 200); + + let _update = index.wait_update_id(1).await; + let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 3); +} From af2cbd0258e93864d646d1f5740fc9253ee4b5ce Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Feb 2021 19:15:42 +0100 Subject: [PATCH 079/522] test get updates --- tests/integration.rs | 1 + tests/updates/mod.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/updates/mod.rs diff --git a/tests/integration.rs b/tests/integration.rs index 34fbde4b6..7e54d8bbf 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -3,6 +3,7 @@ mod index; mod search; mod settings; mod documents; +mod updates; // Tests are isolated by features in different modules to allow better readability, test // targetability, and improved incremental compilation times. diff --git a/tests/updates/mod.rs b/tests/updates/mod.rs new file mode 100644 index 000000000..3fff2d911 --- /dev/null +++ b/tests/updates/mod.rs @@ -0,0 +1,67 @@ +use crate::common::Server; + +#[actix_rt::test] +async fn get_update_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").get_update(0).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn get_unexisting_udpate_status() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (_response, code) = index.get_update(0).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn get_update_status() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + index.add_documents( + serde_json::json!([{ + "id": 1, + "content": "foobar", + }]), + None + ).await; + let (_response, code) = index.get_update(0).await; + assert_eq!(code, 200); + // TODO check resonse format, as per #48 +} + +#[actix_rt::test] +async fn list_updates_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").list_updates().await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn list_no_updates() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (response, code) = index.list_updates().await; + assert_eq!(code, 200); + assert!(response.as_array().unwrap().is_empty()); +} + +// TODO: fix #32 +#[actix_rt::test] +#[ignore] +async fn list_updates() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + index.add_documents( + serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), + None + ).await; + let (response, code) = index.list_updates().await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 1); +} From ac89c35edc361159b653e97c1e0e13516c3cb826 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Feb 2021 19:46:18 +0100 Subject: [PATCH 080/522] add settings routes errors --- src/routes/settings/mod.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/routes/settings/mod.rs b/src/routes/settings/mod.rs index 56c1d34a0..00bc4220e 100644 --- a/src/routes/settings/mod.rs +++ b/src/routes/settings/mod.rs @@ -1,5 +1,4 @@ use actix_web::{web, HttpResponse, delete, get, post}; -use log::error; use crate::Data; use crate::error::ResponseError; @@ -33,8 +32,7 @@ macro_rules! make_setting_route { Ok(HttpResponse::Ok().body(json)) } Err(e) => { - log::error!("{}", e); - unimplemented!(); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -56,8 +54,7 @@ macro_rules! make_setting_route { Ok(HttpResponse::Ok().body(json)) } Err(e) => { - log::error!("{}", e); - unimplemented!(); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -74,8 +71,7 @@ macro_rules! make_setting_route { Ok(HttpResponse::Ok().body(json)) } Err(e) => { - log::error!("{}", e); - unimplemented!(); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -147,8 +143,7 @@ async fn update_all( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("{}", e); - unimplemented!(); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -164,8 +159,7 @@ async fn get_all( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("{}", e); - unimplemented!(); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } @@ -182,8 +176,7 @@ async fn delete_all( Ok(HttpResponse::Ok().body(json)) } Err(e) => { - error!("{}", e); - unimplemented!(); + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } } From c7ab4dccc3a35ab72f01fff138ecf84b6f8e991c Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 09:30:51 +0100 Subject: [PATCH 081/522] test get settings --- tests/common/index.rs | 10 ++++ tests/common/server.rs | 2 +- tests/settings/get_settings.rs | 87 ++++++++++++++++++++++++++++++++++ tests/settings/mod.rs | 1 + 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 tests/settings/get_settings.rs create mode 100644 tests/settings/mod.rs diff --git a/tests/common/index.rs b/tests/common/index.rs index a50f8e6a3..53f48706d 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -131,6 +131,16 @@ impl Index<'_> { let url = format!("/indexes/{}/documents/delete-batch", self.uid); self.service.post(url, serde_json::to_value(&ids).unwrap()).await } + + pub async fn settings(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.service.get(url).await + } + + pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.service.post(url, settings).await + } } pub struct GetDocumentOptions; diff --git a/tests/common/server.rs b/tests/common/server.rs index de32a2045..d7d76445c 100644 --- a/tests/common/server.rs +++ b/tests/common/server.rs @@ -10,7 +10,7 @@ use super::index::Index; use super::service::Service; pub struct Server { - service: Service, + pub service: Service, } impl Server { diff --git a/tests/settings/get_settings.rs b/tests/settings/get_settings.rs new file mode 100644 index 000000000..f66cf03dd --- /dev/null +++ b/tests/settings/get_settings.rs @@ -0,0 +1,87 @@ +use crate::common::Server; +use serde_json::json; + +#[actix_rt::test] +async fn get_settings_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").settings().await; + assert_eq!(code, 400) +} + +#[actix_rt::test] +async fn get_settings() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (response, code) = index.settings().await; + assert_eq!(code, 200); + let settings = response.as_object().unwrap(); + assert_eq!(settings.keys().len(), 3); + assert_eq!(settings["displayedAttributes"], json!(["*"])); + assert_eq!(settings["searchableAttributes"], json!(["*"])); + assert_eq!(settings["facetedAttributes"], json!({})); +} + +#[actix_rt::test] +async fn update_setting_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.update_settings(json!({})).await; + assert_eq!(code, 200); + let (_response, code) = index.get().await; + assert_eq!(code, 200); +} + +macro_rules! test_setting_routes { + ($($setting:ident), *) => { + $( + mod $setting { + use crate::common::Server; + + #[actix_rt::test] + async fn get_unexisting_index() { + let server = Server::new().await; + let url = format!("/indexes/test/settings/{}", + stringify!($setting) + .chars() + .map(|c| if c == '_' { '-' } else { c }) + .collect::()); + let (_response, code) = server.service.get(url).await; + assert_eq!(code, 400); + } + + #[actix_rt::test] + async fn update_unexisting_index() { + let server = Server::new().await; + let url = format!("/indexes/test/settings/{}", + stringify!($setting) + .chars() + .map(|c| if c == '_' { '-' } else { c }) + .collect::()); + let (_response, code) = server.service.post(url, serde_json::Value::Null).await; + assert_eq!(code, 200); + let (_response, code) = server.index("test").get().await; + assert_eq!(code, 200); + } + + #[actix_rt::test] + #[ignore] + async fn delete_unexisting_index() { + let server = Server::new().await; + let url = format!("/indexes/test/settings/{}", + stringify!($setting) + .chars() + .map(|c| if c == '_' { '-' } else { c }) + .collect::()); + let (_response, code) = server.service.delete(url).await; + assert_eq!(code, 400); + } + } + )* + }; +} + +test_setting_routes!( + attributes_for_faceting, + displayed_attributes, + searchable_attributes); diff --git a/tests/settings/mod.rs b/tests/settings/mod.rs new file mode 100644 index 000000000..c9e93c85d --- /dev/null +++ b/tests/settings/mod.rs @@ -0,0 +1 @@ +mod get_settings; From 7d9c5f64aa7768efbe29b143f790b5f8358594ff Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 09:42:36 +0100 Subject: [PATCH 082/522] test partial update --- tests/settings/get_settings.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/settings/get_settings.rs b/tests/settings/get_settings.rs index f66cf03dd..d6d75e389 100644 --- a/tests/settings/get_settings.rs +++ b/tests/settings/get_settings.rs @@ -22,6 +22,34 @@ async fn get_settings() { assert_eq!(settings["facetedAttributes"], json!({})); } +#[actix_rt::test] +async fn update_settings_unknown_field() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.update_settings(json!({"foo": 12})).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn test_partial_update() { + let server = Server::new().await; + let index = server.index("test"); + index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + index.wait_update_id(0).await; + let (response, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!(response["displayedAttributes"],json!(["foo"])); + assert_eq!(response["searchableAttributes"],json!(["*"])); + + index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + index.wait_update_id(1).await; + + let (response, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!(response["displayedAttributes"],json!(["foo"])); + assert_eq!(response["searchableAttributes"],json!(["bar"])); +} + #[actix_rt::test] async fn update_setting_unexisting_index() { let server = Server::new().await; From 3f939f3ccfd0f4b074d33187d00193f7ee6c27cb Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 10:14:36 +0100 Subject: [PATCH 083/522] test delete settings --- tests/common/index.rs | 5 +++++ tests/settings/get_settings.rs | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/common/index.rs b/tests/common/index.rs index 53f48706d..710b96a9f 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -141,6 +141,11 @@ impl Index<'_> { let url = format!("/indexes/{}/settings", self.uid); self.service.post(url, settings).await } + + pub async fn delete_settings(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.service.delete(url).await + } } pub struct GetDocumentOptions; diff --git a/tests/settings/get_settings.rs b/tests/settings/get_settings.rs index d6d75e389..44866c0b0 100644 --- a/tests/settings/get_settings.rs +++ b/tests/settings/get_settings.rs @@ -50,6 +50,16 @@ async fn test_partial_update() { assert_eq!(response["searchableAttributes"],json!(["bar"])); } +#[actix_rt::test] +#[ignore] +// need fix #54 +async fn delete_settings_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.delete_settings().await; + assert_eq!(code, 400); +} + #[actix_rt::test] async fn update_setting_unexisting_index() { let server = Server::new().await; From 60a42bc511d555fbae5ff6ffca766e5d0820072f Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 10:19:22 +0100 Subject: [PATCH 084/522] reset settings --- tests/settings/get_settings.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/settings/get_settings.rs b/tests/settings/get_settings.rs index 44866c0b0..bae044acb 100644 --- a/tests/settings/get_settings.rs +++ b/tests/settings/get_settings.rs @@ -60,6 +60,26 @@ async fn delete_settings_unexisting_index() { assert_eq!(code, 400); } +#[actix_rt::test] +async fn reset_all_settings() { + let server = Server::new().await; + let index = server.index("test"); + index.update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"]})).await; + index.wait_update_id(0).await; + let (response, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!(response["displayedAttributes"],json!(["foo"])); + assert_eq!(response["searchableAttributes"],json!(["bar"])); + + index.delete_settings().await; + index.wait_update_id(1).await; + + let (response, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!(response["displayedAttributes"],json!(["*"])); + assert_eq!(response["searchableAttributes"],json!(["*"])); +} + #[actix_rt::test] async fn update_setting_unexisting_index() { let server = Server::new().await; From caaaf15fd64f6a7de23ed3824ecada837d312202 Mon Sep 17 00:00:00 2001 From: marin Date: Wed, 24 Feb 2021 10:31:28 +0100 Subject: [PATCH 085/522] Create rust.yml --- .github/workflows/rust.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 000000000..3c13d1be2 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,22 @@ +name: Rust + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Build + run: cargo build --verbose + - name: Run tests + run: cargo test --verbose From 61ce749122acf0cf1b8521b19ace855167b8ce74 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Feb 2021 09:10:04 +0100 Subject: [PATCH 086/522] update tokio and disable all routes --- Cargo.lock | 532 +++++++++++++++++++++++++--------- Cargo.toml | 16 +- src/data/mod.rs | 29 +- src/data/search.rs | 150 +++++----- src/data/updates.rs | 100 +++---- src/error.rs | 9 +- src/helpers/authentication.rs | 24 +- src/helpers/normalize_path.rs | 2 +- src/index_controller/mod.rs | 46 ++- src/lib.rs | 52 +--- src/main.rs | 55 +++- src/routes/document.rs | 122 ++++---- src/routes/index.rs | 2 +- src/routes/key.rs | 2 +- 14 files changed, 680 insertions(+), 461 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e4054d87..da8d649dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,8 +12,24 @@ dependencies = [ "futures-sink", "log", "pin-project 0.4.27", - "tokio", - "tokio-util", + "tokio 0.2.24", + "tokio-util 0.3.1", +] + +[[package]] +name = "actix-codec" +version = "0.4.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e" +dependencies = [ + "bitflags", + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", + "tokio 1.2.0", + "tokio-util 0.6.3", ] [[package]] @@ -22,27 +38,22 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-utils", + "actix-codec 0.3.0", + "actix-rt 1.1.1", + "actix-service 1.0.6", + "actix-utils 2.0.0", "derive_more", "either", "futures-util", "http", "log", - "rustls", - "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", - "webpki", ] [[package]] name = "actix-cors" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3a3d5493dbc9b8769fe88c030d057ef8d2edc5728e5e26267780e8fc5db0be" +version = "0.5.4" dependencies = [ "actix-web", "derive_more", @@ -58,28 +69,25 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" dependencies = [ - "actix-codec", + "actix-codec 0.3.0", "actix-connect", - "actix-rt", - "actix-service", + "actix-rt 1.1.1", + "actix-service 1.0.6", "actix-threadpool", - "actix-tls", - "actix-utils", + "actix-utils 2.0.0", "base64 0.13.0", "bitflags", - "brotli2", "bytes 0.5.6", "cookie", "copyless", "derive_more", "either", "encoding_rs", - "flate2", "futures-channel", "futures-core", "futures-util", "fxhash", - "h2", + "h2 0.2.7", "http", "httparse", "indexmap", @@ -100,6 +108,52 @@ dependencies = [ "time 0.2.23", ] +[[package]] +name = "actix-http" +version = "3.0.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a12706e793a92377f85cec219514b72625b3b89f9b4912d8bfb53ab6a615bf0" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-tls", + "actix-utils 3.0.0-beta.2", + "ahash 0.7.0", + "base64 0.13.0", + "bitflags", + "brotli2", + "bytes 1.0.1", + "bytestring", + "cookie", + "derive_more", + "encoding_rs", + "flate2", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.0", + "http", + "httparse", + "indexmap", + "itoa", + "language-tags", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project 1.0.2", + "rand 0.8.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1 0.9.2", + "slab", + "smallvec", + "time 0.2.23", +] + [[package]] name = "actix-macros" version = "0.1.3" @@ -111,10 +165,20 @@ dependencies = [ ] [[package]] -name = "actix-router" -version = "0.2.5" +name = "actix-macros" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd1f7dbda1645bf7da33554db60891755f6c01c1b2169e2f4c492098d30c235" +checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" dependencies = [ "bytestring", "http", @@ -129,33 +193,42 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" dependencies = [ - "actix-macros", + "actix-macros 0.1.3", "actix-threadpool", "copyless", "futures-channel", "futures-util", "smallvec", - "tokio", + "tokio 0.2.24", +] + +[[package]] +name = "actix-rt" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b4e57bc1a3915e71526d128baf4323700bd1580bc676239e2298a4c5b001f18" +dependencies = [ + "actix-macros 0.2.0", + "futures-core", + "tokio 1.2.0", ] [[package]] name = "actix-server" -version = "1.0.4" +version = "2.0.0-beta.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" +checksum = "a99198727204a48f82559c18e4b0ba3197b97d5f4576a32bdbef371f3b4599c1" dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-utils", - "futures-channel", - "futures-util", + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-utils 3.0.0-beta.2", + "futures-core", "log", - "mio", - "mio-uds", + "mio 0.7.9", "num_cpus", "slab", - "socket2", + "tokio 1.2.0", ] [[package]] @@ -169,17 +242,13 @@ dependencies = [ ] [[package]] -name = "actix-testing" -version = "1.0.1" +name = "actix-service" +version = "2.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" +checksum = "ca9756f4d32984ac454ae3155a276f6be69b424197bd3f0ca3c87cde72f41d63" dependencies = [ - "actix-macros", - "actix-rt", - "actix-server", - "actix-service", - "log", - "socket2", + "futures-core", + "pin-project-lite 0.2.0", ] [[package]] @@ -199,18 +268,21 @@ dependencies = [ [[package]] name = "actix-tls" -version = "2.0.0" +version = "3.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24789b7d7361cf5503a504ebe1c10806896f61e96eca9a7350e23001aca715fb" +checksum = "d2b1455e3f7a26d40cfc1080b571f41e8165e5a88e937ed579f7a4b3d55b0370" dependencies = [ - "actix-codec", - "actix-service", - "actix-utils", - "futures-util", - "rustls", - "tokio-rustls", - "webpki", - "webpki-roots", + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-utils 3.0.0-beta.2", + "derive_more", + "futures-core", + "http", + "log", + "tokio-rustls 0.22.0", + "tokio-util 0.6.3", + "webpki-roots 0.21.0", ] [[package]] @@ -219,9 +291,9 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", + "actix-codec 0.3.0", + "actix-rt 1.1.1", + "actix-service 1.0.6", "bitflags", "bytes 0.5.6", "either", @@ -234,50 +306,63 @@ dependencies = [ ] [[package]] -name = "actix-web" -version = "3.3.2" +name = "actix-utils" +version = "3.0.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86" +checksum = "458795e09a29bc5557604f9ff6f32236fd0ee457d631672e4ec8f6a0103bb292" dependencies = [ - "actix-codec", - "actix-http", - "actix-macros", + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", +] + +[[package]] +name = "actix-web" +version = "4.0.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9683dc8c3037ea524e0fec6032d34e1cb1ee72c4eb8689f428a60c2a544ea3" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-http 3.0.0-beta.3", + "actix-macros 0.2.0", "actix-router", - "actix-rt", + "actix-rt 2.1.0", "actix-server", - "actix-service", - "actix-testing", - "actix-threadpool", + "actix-service 2.0.0-beta.4", "actix-tls", - "actix-utils", + "actix-utils 3.0.0-beta.2", "actix-web-codegen", + "ahash 0.7.0", "awc", - "bytes 0.5.6", + "bytes 1.0.1", "derive_more", + "either", "encoding_rs", - "futures-channel", "futures-core", "futures-util", - "fxhash", "log", "mime", "pin-project 1.0.2", "regex", - "rustls", + "rustls 0.19.0", "serde", "serde_json", "serde_urlencoded", + "smallvec", "socket2", "time 0.2.23", - "tinyvec", "url", ] [[package]] name = "actix-web-codegen" -version = "0.4.0" +version = "0.5.0-beta.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad26f77093333e0e7c6ffe54ebe3582d908a104e448723eec6d43d08b07143fb" +checksum = "8313dc4cbcae1785a7f14c3dfb7dfeb25fe96a03b20e5c38fe026786def5aa70" dependencies = [ "proc-macro2", "quote", @@ -305,6 +390,17 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +[[package]] +name = "ahash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa60d2eadd8b12a996add391db32bd1153eac697ba4869660c0016353611426" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.15" @@ -348,7 +444,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite 0.1.11", - "tokio", + "tokio 0.2.24", ] [[package]] @@ -381,24 +477,24 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "awc" -version = "2.0.3" +version = "3.0.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" +checksum = "da7225ad81fbad09ef56ccc61e0688abe8494a68722c5d0df5e2fc8b724a200b" dependencies = [ - "actix-codec", - "actix-http", - "actix-rt", - "actix-service", + "actix-codec 0.4.0-beta.1", + "actix-http 3.0.0-beta.3", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", "base64 0.13.0", - "bytes 0.5.6", + "bytes 1.0.1", "cfg-if 1.0.0", "derive_more", "futures-core", "log", "mime", "percent-encoding", - "rand 0.7.3", - "rustls", + "rand 0.8.3", + "rustls 0.19.0", "serde", "serde_json", "serde_urlencoded", @@ -563,12 +659,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" [[package]] -name = "bytestring" -version = "0.1.5" +name = "bytes" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7c05fa5172da78a62d9949d662d2ac89d4cc7355d7b49adee5163f1fb3f363" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "bytestring" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", ] [[package]] @@ -1171,8 +1273,28 @@ dependencies = [ "http", "indexmap", "slab", - "tokio", - "tokio-util", + "tokio 0.2.24", + "tokio-util 0.3.1", + "tracing", + "tracing-futures", +] + +[[package]] +name = "h2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.2.0", + "tokio-util 0.6.3", "tracing", "tracing-futures", ] @@ -1183,7 +1305,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" dependencies = [ - "ahash", + "ahash 0.3.8", "autocfg", ] @@ -1262,11 +1384,11 @@ dependencies = [ [[package]] name = "http" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84129d298a6d57d246960ff8eb831ca4af3f96d29e2e28848dae275408658e26" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "fnv", "itoa", ] @@ -1324,7 +1446,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.2.7", "http", "http-body", "httparse", @@ -1332,7 +1454,7 @@ dependencies = [ "itoa", "pin-project 1.0.2", "socket2", - "tokio", + "tokio 0.2.24", "tower-service", "tracing", "want", @@ -1348,9 +1470,9 @@ dependencies = [ "futures-util", "hyper", "log", - "rustls", - "tokio", - "tokio-rustls", + "rustls 0.18.1", + "tokio 0.2.24", + "tokio-rustls 0.14.1", "webpki", ] @@ -1537,9 +1659,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" [[package]] name = "linked-hash-map" @@ -1610,24 +1732,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] -name = "meilisearch-error" -version = "0.19.0" -dependencies = [ - "actix-http", -] - -[[package]] -name = "meilisearch-http" +name = "meilisearch" version = "0.17.0" dependencies = [ "actix-cors", - "actix-http", - "actix-rt", - "actix-service", + "actix-service 1.0.6", "actix-web", "anyhow", "assert-json-diff", "async-compression", + "async-trait", "byte-unit", "bytes 0.6.0", "chrono", @@ -1656,7 +1770,7 @@ dependencies = [ "rand 0.7.3", "rayon", "regex", - "rustls", + "rustls 0.19.0", "sentry", "serde", "serde_json", @@ -1668,11 +1782,20 @@ dependencies = [ "tar", "tempdir", "tempfile", - "tokio", + "thiserror", + "tokio 0.2.24", + "tokio 1.2.0", "uuid", "vergen", ] +[[package]] +name = "meilisearch-error" +version = "0.19.0" +dependencies = [ + "actix-http 2.2.0", +] + [[package]] name = "meilisearch-tokenizer" version = "0.1.1" @@ -1799,14 +1922,15 @@ dependencies = [ ] [[package]] -name = "mio-named-pipes" -version = "0.1.7" +name = "mio" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" dependencies = [ + "libc", "log", - "mio", "miow 0.3.6", + "ntapi", "winapi 0.3.9", ] @@ -1818,7 +1942,7 @@ checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" dependencies = [ "iovec", "libc", - "mio", + "mio 0.6.23", ] [[package]] @@ -1866,6 +1990,15 @@ dependencies = [ "libc", ] +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -2215,12 +2348,24 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom 0.1.15", "libc", - "rand_chacha", + "rand_chacha 0.2.2", "rand_core 0.5.1", - "rand_hc", + "rand_hc 0.2.0", "rand_pcg", ] +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -2231,6 +2376,16 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -2255,6 +2410,15 @@ dependencies = [ "getrandom 0.1.15", ] +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -2264,6 +2428,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + [[package]] name = "rand_pcg" version = "0.2.1" @@ -2381,17 +2554,17 @@ dependencies = [ "mime_guess", "percent-encoding", "pin-project-lite 0.2.0", - "rustls", + "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", - "tokio", - "tokio-rustls", + "tokio 0.2.24", + "tokio-rustls 0.14.1", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.20.0", "winreg 0.7.0", ] @@ -2457,6 +2630,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.5" @@ -2680,9 +2866,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" @@ -2883,18 +3069,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ "proc-macro2", "quote", @@ -2996,14 +3182,33 @@ dependencies = [ "lazy_static", "libc", "memchr", - "mio", - "mio-named-pipes", + "mio 0.6.23", "mio-uds", "num_cpus", "pin-project-lite 0.1.11", "signal-hook-registry", "slab", - "tokio-macros", + "tokio-macros 0.2.6", + "winapi 0.3.9", +] + +[[package]] +name = "tokio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +dependencies = [ + "autocfg", + "bytes 1.0.1", + "libc", + "memchr", + "mio 0.7.9", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite 0.2.0", + "signal-hook-registry", + "tokio-macros 1.1.0", "winapi 0.3.9", ] @@ -3018,6 +3223,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-rustls" version = "0.14.1" @@ -3025,8 +3241,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", - "rustls", - "tokio", + "rustls 0.18.1", + "tokio 0.2.24", + "webpki", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.0", + "tokio 1.2.0", "webpki", ] @@ -3041,7 +3268,21 @@ dependencies = [ "futures-sink", "log", "pin-project-lite 0.1.11", - "tokio", + "tokio 0.2.24", +] + +[[package]] +name = "tokio-util" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", + "tokio 1.2.0", ] [[package]] @@ -3097,7 +3338,7 @@ dependencies = [ "rand 0.7.3", "smallvec", "thiserror", - "tokio", + "tokio 0.2.24", "url", ] @@ -3117,7 +3358,7 @@ dependencies = [ "resolv-conf", "smallvec", "thiserror", - "tokio", + "tokio 0.2.24", "trust-dns-proto", ] @@ -3369,6 +3610,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +dependencies = [ + "webpki", +] + [[package]] name = "whatlang" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 993c80946..126f859e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Quentin de Quelen ", "Clément Renault , + pub index_controller: Arc, pub api_keys: ApiKeys, options: Opt, } @@ -59,14 +60,9 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - let indexer_opts = options.indexer_options.clone(); + //let indexer_opts = options.indexer_options.clone(); create_dir_all(&path)?; - let index_controller = LocalIndexController::new( - &path, - indexer_opts, - options.max_mdb_size.get_bytes(), - options.max_udb_size.get_bytes(), - )?; + let index_controller = ActorIndexController::new(); let index_controller = Arc::new(index_controller); let mut api_keys = ApiKeys { @@ -85,7 +81,7 @@ impl Data { pub fn settings>(&self, index_uid: S) -> anyhow::Result { let index = self.index_controller - .index(&index_uid)? + .index(index_uid.as_ref().to_string())? .ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?; let txn = index.read_txn()?; @@ -119,19 +115,20 @@ impl Data { } pub fn index(&self, name: impl AsRef) -> anyhow::Result> { - Ok(self - .list_indexes()? - .into_iter() - .find(|i| i.uid == name.as_ref())) + todo!() + //Ok(self + //.list_indexes()? + //.into_iter() + //.find(|i| i.uid == name.as_ref())) } - pub fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { + pub async fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { let settings = IndexSettings { name: Some(name.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), }; - let meta = self.index_controller.create_index(settings)?; + let meta = self.index_controller.create_index(settings).await?; Ok(meta) } diff --git a/src/data/search.rs b/src/data/search.rs index f26730fcf..23c6b463e 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -2,7 +2,7 @@ use std::collections::{HashSet, BTreeMap}; use std::mem; use std::time::Instant; -use anyhow::{bail, Context}; +use anyhow::bail; use either::Either; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use super::Data; -use crate::index_controller::IndexController; pub const DEFAULT_SEARCH_LIMIT: usize = 20; @@ -202,107 +201,110 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { impl Data { pub fn search>( &self, - index: S, - search_query: SearchQuery, + _index: S, + _search_query: SearchQuery, ) -> anyhow::Result { - match self.index_controller.index(&index)? { - Some(index) => Ok(search_query.perform(index)?), - None => bail!("index {:?} doesn't exists", index.as_ref()), - } + todo!() + //match self.index_controller.index(&index)? { + //Some(index) => Ok(search_query.perform(index)?), + //None => bail!("index {:?} doesn't exists", index.as_ref()), + //} } pub async fn retrieve_documents( &self, - index: impl AsRef + Send + Sync + 'static, - offset: usize, - limit: usize, - attributes_to_retrieve: Option>, + _index: String, + _offset: usize, + _limit: usize, + _attributes_to_retrieve: Option>, ) -> anyhow::Result>> where S: AsRef + Send + Sync + 'static, { - let index_controller = self.index_controller.clone(); - let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { - let index = index_controller - .index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + todo!() + //let index_controller = self.index_controller.clone(); + //let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { + //let index = index_controller + //.index(index.clone())? + //.with_context(|| format!("Index {:?} doesn't exist", index))?; - let txn = index.read_txn()?; + //let txn = index.read_txn()?; - let fields_ids_map = index.fields_ids_map(&txn)?; + //let fields_ids_map = index.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + //let attributes_to_retrieve_ids = match attributes_to_retrieve { + //Some(attrs) => attrs + //.iter() + //.filter_map(|f| fields_ids_map.id(f.as_ref())) + //.collect::>(), + //None => fields_ids_map.iter().map(|(id, _)| id).collect(), + //}; - let iter = index.documents.range(&txn, &(..))?.skip(offset).take(limit); + //let iter = index.documents.range(&txn, &(..))?.skip(offset).take(limit); - let mut documents = Vec::new(); + //let mut documents = Vec::new(); - for entry in iter { - let (_id, obkv) = entry?; - let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; - documents.push(object); - } + //for entry in iter { + //let (_id, obkv) = entry?; + //let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; + //documents.push(object); + //} - Ok(documents) - }) - .await?; - documents + //Ok(documents) + //}) + //.await?; + //documents } pub async fn retrieve_document( &self, - index: impl AsRef + Sync + Send + 'static, - document_id: impl AsRef + Sync + Send + 'static, - attributes_to_retrieve: Option>, + _index: impl AsRef + Sync + Send + 'static, + _document_id: impl AsRef + Sync + Send + 'static, + _attributes_to_retrieve: Option>, ) -> anyhow::Result> where S: AsRef + Sync + Send + 'static, { - let index_controller = self.index_controller.clone(); - let document: anyhow::Result<_> = tokio::task::spawn_blocking(move || { - let index = index_controller - .index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let txn = index.read_txn()?; + todo!() + //let index_controller = self.index_controller.clone(); + //let document: anyhow::Result<_> = tokio::task::spawn_blocking(move || { + //let index = index_controller + //.index(&index)? + //.with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + //let txn = index.read_txn()?; - let fields_ids_map = index.fields_ids_map(&txn)?; + //let fields_ids_map = index.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + //let attributes_to_retrieve_ids = match attributes_to_retrieve { + //Some(attrs) => attrs + //.iter() + //.filter_map(|f| fields_ids_map.id(f.as_ref())) + //.collect::>(), + //None => fields_ids_map.iter().map(|(id, _)| id).collect(), + //}; - let internal_id = index - .external_documents_ids(&txn)? - .get(document_id.as_ref().as_bytes()) - .with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; + //let internal_id = index + //.external_documents_ids(&txn)? + //.get(document_id.as_ref().as_bytes()) + //.with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; - let document = index - .documents(&txn, std::iter::once(internal_id))? - .into_iter() - .next() - .map(|(_, d)| d); + //let document = index + //.documents(&txn, std::iter::once(internal_id))? + //.into_iter() + //.next() + //.map(|(_, d)| d); - match document { - Some(document) => Ok(obkv_to_json( - &attributes_to_retrieve_ids, - &fields_ids_map, - document, - )?), - None => bail!("Document with id {} not found", document_id.as_ref()), - } - }) - .await?; - document + //match document { + //Some(document) => Ok(obkv_to_json( + //&attributes_to_retrieve_ids, + //&fields_ids_map, + //document, + //)?), + //None => bail!("Document with id {} not found", document_id.as_ref()), + //} + //}) + //.await?; + //document } } diff --git a/src/data/updates.rs b/src/data/updates.rs index fbb9be801..5cdaf7db1 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,12 +1,13 @@ use std::ops::Deref; -use async_compression::tokio_02::write::GzipEncoder; -use futures_util::stream::StreamExt; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use tokio::io::AsyncWriteExt; +//use async_compression::tokio_02::write::GzipEncoder; +//use futures_util::stream::StreamExt; +//use milli::update::{IndexDocumentsMethod, UpdateFormat}; +//use tokio::io::AsyncWriteExt; +use actix_web::web::Payload; use crate::index_controller::UpdateStatus; -use crate::index_controller::{IndexController, Settings, IndexSettings, IndexMetadata}; +use crate::index_controller::{Settings, IndexMetadata}; use super::Data; impl Data { @@ -15,88 +16,68 @@ impl Data { index: impl AsRef + Send + Sync + 'static, method: IndexDocumentsMethod, format: UpdateFormat, - mut stream: impl futures::Stream> + Unpin, + stream: Payload, primary_key: Option, ) -> anyhow::Result where B: Deref, E: std::error::Error + Send + Sync + 'static, { - let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; - let file = tokio::fs::File::from_std(file?); - let mut encoder = GzipEncoder::new(file); - - let mut empty_update = true; - while let Some(result) = stream.next().await { - empty_update = false; - let bytes = &*result?; - encoder.write_all(&bytes[..]).await?; - } - - encoder.shutdown().await?; - let mut file = encoder.into_inner(); - file.sync_all().await?; - let file = file.into_std().await; - - let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move ||{ - let mmap; - let bytes = if empty_update { - &[][..] - } else { - mmap = unsafe { memmap::Mmap::map(&file)? }; - &mmap - }; - index_controller.add_documents(index, method, format, &bytes, primary_key) - }).await??; - Ok(update.into()) + let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, stream, primary_key).await?; + Ok(update_status) } pub async fn update_settings( &self, - index: impl AsRef + Send + Sync + 'static, - settings: Settings + _index: impl AsRef + Send + Sync + 'static, + _settings: Settings ) -> anyhow::Result { - let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; - Ok(update.into()) + todo!() + //let index_controller = self.index_controller.clone(); + //let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; + //Ok(update.into()) } pub async fn clear_documents( &self, - index: impl AsRef + Sync + Send + 'static, + _index: impl AsRef + Sync + Send + 'static, ) -> anyhow::Result { - let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.clear_documents(index)).await??; - Ok(update.into()) + todo!() + //let index_controller = self.index_controller.clone(); + //let update = tokio::task::spawn_blocking(move || index_controller.clear_documents(index)).await??; + //Ok(update.into()) } pub async fn delete_documents( &self, - index: impl AsRef + Sync + Send + 'static, - document_ids: Vec, + _index: impl AsRef + Sync + Send + 'static, + _document_ids: Vec, ) -> anyhow::Result { - let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; - Ok(update.into()) + todo!() + //let index_controller = self.index_controller.clone(); + //let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; + //Ok(update.into()) } pub async fn delete_index( &self, - index: impl AsRef + Send + Sync + 'static, + _index: impl AsRef + Send + Sync + 'static, ) -> anyhow::Result<()> { - let index_controller = self.index_controller.clone(); - tokio::task::spawn_blocking(move || { index_controller.delete_index(index) }).await??; - Ok(()) + todo!() + //let index_controller = self.index_controller.clone(); + //tokio::task::spawn_blocking(move || { index_controller.delete_index(index) }).await??; + //Ok(()) } #[inline] pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { - self.index_controller.update_status(index, uid) + todo!() + //self.index_controller.update_status(index, uid) } pub fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { - self.index_controller.all_update_status(index) + todo!() + //self.index_controller.all_update_status(index) } pub fn update_index( @@ -105,11 +86,12 @@ impl Data { primary_key: Option>, new_name: Option> ) -> anyhow::Result { - let settings = IndexSettings { - name: new_name.map(|s| s.as_ref().to_string()), - primary_key: primary_key.map(|s| s.as_ref().to_string()), - }; + todo!() + //let settings = IndexSettings { + //name: new_name.map(|s| s.as_ref().to_string()), + //primary_key: primary_key.map(|s| s.as_ref().to_string()), + //}; - self.index_controller.update_index(name, settings) + //self.index_controller.update_index(name, settings) } } diff --git a/src/error.rs b/src/error.rs index 33aa06d3c..70e9c4ed8 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,7 +1,8 @@ use std::error; use std::fmt; -use actix_http::ResponseBuilder; +use actix_web::dev::HttpResponseBuilder; +use actix_web::http::Error as HttpError; use actix_web as aweb; use actix_web::error::{JsonPayloadError, QueryPayloadError}; use actix_web::http::StatusCode; @@ -66,7 +67,7 @@ impl Serialize for ResponseError { impl aweb::error::ResponseError for ResponseError { fn error_response(&self) -> aweb::HttpResponse { - ResponseBuilder::new(self.status_code()).json(&self) + HttpResponseBuilder::new(self.status_code()).json(&self) } fn status_code(&self) -> StatusCode { @@ -260,8 +261,8 @@ impl From for Error { } } -impl From for Error { - fn from(err: actix_http::Error) -> Error { +impl From for Error { + fn from(err: HttpError) -> Error { Error::Internal(err.to_string()) } } diff --git a/src/helpers/authentication.rs b/src/helpers/authentication.rs index 0de709ba3..4fdbdeef5 100644 --- a/src/helpers/authentication.rs +++ b/src/helpers/authentication.rs @@ -3,27 +3,26 @@ use std::pin::Pin; use std::rc::Rc; use std::task::{Context, Poll}; -use actix_service::{Service, Transform}; -use actix_web::{dev::ServiceRequest, dev::ServiceResponse, web}; +use actix_web::dev::{Transform, Service, ServiceResponse, ServiceRequest}; +use actix_web::web; use futures::future::{err, ok, Future, Ready}; use crate::error::{Error, ResponseError}; use crate::Data; -#[derive(Clone)] +#[derive(Clone, Copy)] pub enum Authentication { Public, Private, Admin, } -impl Transform for Authentication +impl Transform for Authentication where - S: Service, Error = actix_web::Error>, + S: Service, Error = actix_web::Error>, S::Future: 'static, B: 'static, { - type Request = ServiceRequest; type Response = ServiceResponse; type Error = actix_web::Error; type InitError = (); @@ -32,7 +31,7 @@ where fn new_transform(&self, service: S) -> Self::Future { ok(LoggingMiddleware { - acl: self.clone(), + acl: *self, service: Rc::new(RefCell::new(service)), }) } @@ -44,23 +43,22 @@ pub struct LoggingMiddleware { } #[allow(clippy::type_complexity)] -impl Service for LoggingMiddleware +impl Service for LoggingMiddleware where - S: Service, Error = actix_web::Error> + 'static, + S: Service, Error = actix_web::Error> + 'static, S::Future: 'static, B: 'static, { - type Request = ServiceRequest; type Response = ServiceResponse; type Error = actix_web::Error; type Future = Pin>>>; - fn poll_ready(&mut self, cx: &mut Context) -> Poll> { + fn poll_ready(&self, cx: &mut Context) -> Poll> { self.service.poll_ready(cx) } - fn call(&mut self, req: ServiceRequest) -> Self::Future { - let mut svc = self.service.clone(); + fn call(&self, req: ServiceRequest) -> Self::Future { + let svc = self.service.clone(); // This unwrap is left because this error should never appear. If that's the case, then // it means that actix-web has an issue or someone changes the type `Data`. let data = req.app_data::>().unwrap(); diff --git a/src/helpers/normalize_path.rs b/src/helpers/normalize_path.rs index e669b9d94..51c64c52b 100644 --- a/src/helpers/normalize_path.rs +++ b/src/helpers/normalize_path.rs @@ -1,5 +1,5 @@ /// From https://docs.rs/actix-web/3.0.0-alpha.2/src/actix_web/middleware/normalize.rs.html#34 -use actix_http::Error; +use actix_web::http::Error; use actix_service::{Service, Transform}; use actix_web::{ dev::ServiceRequest, diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index b20e43749..ed085456b 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,8 +1,7 @@ -mod local_index_controller; +pub mod actor_index_controller; +//mod local_index_controller; mod updates; -pub use local_index_controller::LocalIndexController; - use std::collections::HashMap; use std::num::NonZeroUsize; use std::sync::Arc; @@ -13,6 +12,7 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; +use actix_web::web::Payload; pub use updates::{Processed, Processing, Failed}; @@ -21,7 +21,6 @@ pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { - pub uid: String, uuid: Uuid, created_at: DateTime, updated_at: DateTime, @@ -114,6 +113,7 @@ pub struct IndexSettings { /// be provided. This allows the implementer to define the behaviour of write accesses to the /// indices, and abstract the scheduling of the updates. The implementer must be able to provide an /// instance of `IndexStore` +#[async_trait::async_trait] pub trait IndexController { /* @@ -126,59 +126,47 @@ pub trait IndexController { /// Perform document addition on the database. If the provided index does not exist, it will be /// created when the addition is applied to the index. - fn add_documents>( + async fn add_documents( &self, - index: S, + index: String, method: IndexDocumentsMethod, format: UpdateFormat, - data: &[u8], + data: Payload, primary_key: Option, ) -> anyhow::Result; /// Clear all documents in the given index. - fn clear_documents(&self, index: impl AsRef) -> anyhow::Result; + fn clear_documents(&self, index: String) -> anyhow::Result; /// Delete all documents in `document_ids`. - fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result; + fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result; /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. - fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; + fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result; /// Create an index with the given `index_uid`. - fn create_index(&self, index_settings: IndexSettings) -> Result; + async fn create_index(&self, index_settings: IndexSettings) -> Result; /// Delete index with the given `index_uid`, attempting to close it beforehand. - fn delete_index>(&self, index_uid: S) -> Result<()>; + fn delete_index(&self, index_uid: String) -> Result<()>; /// Swap two indexes, concretely, it simply swaps the index the names point to. - fn swap_indices, S2: AsRef>(&self, index1_uid: S1, index2_uid: S2) -> Result<()>; - - /// Apply an update to the given index. This method can be called when an update is ready to be - /// processed - fn handle_update>( - &self, - _index: S, - _update_id: u64, - _meta: Processing, - _content: &[u8] - ) -> Result, Failed> { - todo!() - } + fn swap_indices(&self, index1_uid: String, index2_uid: String) -> Result<()>; /// Returns, if it exists, the `Index` with the povided name. - fn index(&self, name: impl AsRef) -> anyhow::Result>>; + fn index(&self, name: String) -> anyhow::Result>>; /// Returns the udpate status an update - fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>; + fn update_status(&self, index: String, id: u64) -> anyhow::Result>; /// Returns all the udpate status for an index - fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>; + fn all_update_status(&self, index: String) -> anyhow::Result>; /// List all the indexes fn list_indexes(&self) -> anyhow::Result>; - fn update_index(&self, name: impl AsRef, index_settings: IndexSettings) -> anyhow::Result; + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result; } diff --git a/src/lib.rs b/src/lib.rs index d542fd6d7..65c4e8eb7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,7 @@ #![allow(clippy::or_fun_call)] +#![allow(unused_must_use)] +#![allow(unused_variables)] +#![allow(dead_code)] pub mod data; pub mod error; @@ -7,54 +10,5 @@ pub mod option; pub mod routes; mod index_controller; -use actix_http::Error; -use actix_service::ServiceFactory; -use actix_web::{dev, web, App}; - pub use option::Opt; pub use self::data::Data; -use self::error::payload_error_handler; - -pub fn create_app( - data: &Data, - enable_frontend: bool, -) -> App< - impl ServiceFactory< - Config = (), - Request = dev::ServiceRequest, - Response = dev::ServiceResponse, - Error = Error, - InitError = (), - >, - actix_http::body::Body, -> { - let app = App::new() - .data(data.clone()) - .app_data( - web::JsonConfig::default() - .limit(data.http_payload_size_limit()) - .content_type(|_mime| true) // Accept all mime types - .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .app_data( - web::QueryConfig::default() - .error_handler(|err, _req| payload_error_handler(err).into()) - ) - .configure(routes::document::services) - .configure(routes::index::services) - .configure(routes::search::services) - .configure(routes::settings::services) - .configure(routes::stop_words::services) - .configure(routes::synonym::services) - .configure(routes::health::services) - .configure(routes::stats::services) - .configure(routes::key::services); - //.configure(routes::dump::services); - if enable_frontend { - app - .service(routes::load_html) - .service(routes::load_css) - } else { - app - } -} diff --git a/src/main.rs b/src/main.rs index 7e50225be..c9042b88e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,13 @@ use std::env; use actix_cors::Cors; -use actix_web::{middleware, HttpServer}; +use actix_web::{middleware, HttpServer, web, web::ServiceConfig}; use main_error::MainError; -use meilisearch_http::helpers::NormalizePath; -use meilisearch_http::{create_app, Data, Opt}; +use meilisearch::{Data, Opt}; use structopt::StructOpt; +use actix_web::App; +use meilisearch::error::payload_error_handler; +use actix_web::middleware::TrailingSlash; //mod analytics; @@ -74,9 +76,34 @@ async fn main() -> Result<(), MainError> { print_launch_resume(&opt, &data); let enable_frontend = opt.env != "production"; + + run_http(data, opt, enable_frontend).await?; + + Ok(()) +} + +async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box> { let http_server = HttpServer::new(move || { - create_app(&data, enable_frontend) - .wrap( + let app = App::new() + .configure(|c| configure_data(c, &data)) + .configure(meilisearch::routes::document::services) + .configure(meilisearch::routes::index::services) + .configure(meilisearch::routes::search::services) + .configure(meilisearch::routes::settings::services) + .configure(meilisearch::routes::stop_words::services) + .configure(meilisearch::routes::synonym::services) + .configure(meilisearch::routes::health::services) + .configure(meilisearch::routes::stats::services) + .configure(meilisearch::routes::key::services); + //.configure(routes::dump::services); + let app = if enable_frontend { + app + .service(meilisearch::routes::load_html) + .service(meilisearch::routes::load_css) + } else { + app + }; + app.wrap( Cors::default() .send_wildcard() .allowed_headers(vec!["content-type", "x-meili-api-key"]) @@ -84,7 +111,7 @@ async fn main() -> Result<(), MainError> { ) .wrap(middleware::Logger::default()) .wrap(middleware::Compress::default()) - .wrap(NormalizePath) + .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) }); if let Some(config) = opt.get_ssl_config()? { @@ -95,10 +122,24 @@ async fn main() -> Result<(), MainError> { } else { http_server.bind(opt.http_addr)?.run().await?; } - Ok(()) } +fn configure_data(config: &mut ServiceConfig, data: &Data) { + config + .data(data.clone()) + .app_data( + web::JsonConfig::default() + .limit(data.http_payload_size_limit()) + .content_type(|_mime| true) // Accept all mime types + .error_handler(|err, _req| payload_error_handler(err).into()), + ) + .app_data( + web::QueryConfig::default() + .error_handler(|err, _req| payload_error_handler(err).into()) + ); +} + pub fn print_launch_resume(opt: &Opt, data: &Data) { let ascii_name = r#" 888b d888 d8b 888 d8b .d8888b. 888 diff --git a/src/routes/document.rs b/src/routes/document.rs index 43d88e4b9..2cb87074e 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -3,7 +3,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use indexmap::IndexMap; use log::error; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; +//use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; use serde_json::Value; @@ -142,25 +142,26 @@ async fn add_documents_json( params: web::Query, body: Payload, ) -> Result { - let addition_result = data - .add_documents( - path.into_inner().index_uid, - IndexDocumentsMethod::ReplaceDocuments, - UpdateFormat::Json, - body, - params.primary_key.clone(), - ).await; + todo!() + //let addition_result = data + //.add_documents( + //path.into_inner().index_uid, + //IndexDocumentsMethod::ReplaceDocuments, + //UpdateFormat::Json, + //body, + //params.primary_key.clone(), + //).await; - match addition_result { - Ok(update) => { - let value = serde_json::to_string(&update).unwrap(); - let response = HttpResponse::Ok().body(value); - Ok(response) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + //match addition_result { + //Ok(update) => { + //let value = serde_json::to_string(&update).unwrap(); + //let response = HttpResponse::Ok().body(value); + //Ok(response) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } @@ -199,25 +200,26 @@ async fn update_documents( params: web::Query, body: web::Payload, ) -> Result { - let addition_result = data - .add_documents( - path.into_inner().index_uid, - IndexDocumentsMethod::UpdateDocuments, - UpdateFormat::Json, - body, - params.primary_key.clone(), - ).await; + todo!() + //let addition_result = data + //.add_documents( + //path.into_inner().index_uid, + //IndexDocumentsMethod::UpdateDocuments, + //UpdateFormat::Json, + //body, + //params.primary_key.clone(), + //).await; - match addition_result { - Ok(update) => { - let value = serde_json::to_string(&update).unwrap(); - let response = HttpResponse::Ok().body(value); - Ok(response) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + //match addition_result { + //Ok(update) => { + //let value = serde_json::to_string(&update).unwrap(); + //let response = HttpResponse::Ok().body(value); + //Ok(response) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } #[post( @@ -229,20 +231,21 @@ async fn delete_documents( path: web::Path, body: web::Json>, ) -> Result { - let ids = body - .iter() - .map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) - .collect(); + todo!() + //let ids = body + //.iter() + //.map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) + //.collect(); - match data.delete_documents(path.index_uid.clone(), ids).await { - Ok(result) => { - let json = serde_json::to_string(&result).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + //match data.delete_documents(path.index_uid.clone(), ids).await { + //Ok(result) => { + //let json = serde_json::to_string(&result).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] @@ -250,13 +253,14 @@ async fn clear_all_documents( data: web::Data, path: web::Path, ) -> Result { - match data.clear_documents(path.index_uid.clone()).await { - Ok(update) => { - let json = serde_json::to_string(&update).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + todo!() + //match data.clear_documents(path.index_uid.clone()).await { + //Ok(update) => { + //let json = serde_json::to_string(&update).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } diff --git a/src/routes/index.rs b/src/routes/index.rs index 2c478432b..813256517 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -61,7 +61,7 @@ async fn create_index( data: web::Data, body: web::Json, ) -> Result { - match data.create_index(&body.uid, body.primary_key.clone()) { + match data.create_index(&body.uid, body.primary_key.clone()).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) diff --git a/src/routes/key.rs b/src/routes/key.rs index a0cbaccc3..a051ee0f9 100644 --- a/src/routes/key.rs +++ b/src/routes/key.rs @@ -19,7 +19,7 @@ struct KeysResponse { #[get("/keys", wrap = "Authentication::Admin")] async fn list(data: web::Data) -> HttpResponse { let api_keys = data.api_keys.clone(); - HttpResponse::Ok().json(KeysResponse { + HttpResponse::Ok().json(&KeysResponse { private: api_keys.private, public: api_keys.public, }) From 672a4b54005cca57a737117ac2e7631f910f5a4b Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Feb 2021 09:10:36 +0100 Subject: [PATCH 087/522] add actors/ support index creation --- .../actor_index_controller/index_actor.rs | 132 ++++++++++++++++++ .../actor_index_controller/mod.rs | 93 ++++++++++++ .../actor_index_controller/update_actor.rs | 16 +++ .../actor_index_controller/uuid_resolver.rs | 116 +++++++++++++++ 4 files changed, 357 insertions(+) create mode 100644 src/index_controller/actor_index_controller/index_actor.rs create mode 100644 src/index_controller/actor_index_controller/mod.rs create mode 100644 src/index_controller/actor_index_controller/update_actor.rs create mode 100644 src/index_controller/actor_index_controller/uuid_resolver.rs diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs new file mode 100644 index 000000000..6de0f2b77 --- /dev/null +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -0,0 +1,132 @@ +use uuid::Uuid; +use std::path::{PathBuf, Path}; +use chrono::Utc; +use tokio::sync::{mpsc, oneshot, RwLock}; +use thiserror::Error; +use std::collections::HashMap; +use std::sync::Arc; +use milli::Index; +use std::collections::hash_map::Entry; +use std::fs::create_dir_all; +use heed::EnvOpenOptions; +use crate::index_controller::IndexMetadata; + +pub type Result = std::result::Result; +type AsyncMap = Arc>>; + +enum IndexMsg { + CreateIndex { uuid: Uuid, primary_key: Option, ret: oneshot::Sender> }, +} + +struct IndexActor { + inbox: mpsc::Receiver, + store: S, +} + +#[derive(Error, Debug)] +pub enum IndexError { + #[error("error with index: {0}")] + Error(#[from] anyhow::Error), + #[error("index already exists")] + IndexAlreadyExists, +} + +#[async_trait::async_trait] +trait IndexStore { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; +} + +impl IndexActor { + fn new(inbox: mpsc::Receiver, store: S) -> Self { + Self { inbox, store } + } + + async fn run(mut self) { + loop { + match self.inbox.recv().await { + Some(IndexMsg::CreateIndex { uuid, primary_key, ret }) => self.handle_create_index(uuid, primary_key, ret).await, + None => break, + } + } + } + + async fn handle_create_index(&self, uuid: Uuid, primary_key: Option, ret: oneshot::Sender>) { + let result = self.store.create_index(uuid, primary_key).await; + let _ = ret.send(result); + } +} + +#[derive(Clone)] +pub struct IndexActorHandle { + sender: mpsc::Sender, +} + +impl IndexActorHandle { + pub fn new() -> Self { + let (sender, receiver) = mpsc::channel(100); + + let store = MapIndexStore::new("data.ms"); + let actor = IndexActor::new(receiver, store); + tokio::task::spawn(actor.run()); + Self { sender } + } + + pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::CreateIndex { ret, uuid, primary_key }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } +} + +struct MapIndexStore { + root: PathBuf, + meta_store: AsyncMap, + index_store: AsyncMap, +} + +#[async_trait::async_trait] +impl IndexStore for MapIndexStore { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + let meta = match self.meta_store.write().await.entry(uuid.clone()) { + Entry::Vacant(entry) => { + let meta = IndexMetadata { + uuid, + created_at: Utc::now(), + updated_at: Utc::now(), + primary_key, + }; + entry.insert(meta).clone() + } + Entry::Occupied(_) => return Err(IndexError::IndexAlreadyExists), + }; + + let db_path = self.root.join(format!("index-{}", meta.uuid)); + + + println!("before blocking"); + let index: Result = tokio::task::spawn_blocking(move || { + create_dir_all(&db_path).expect("can't create db"); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let index = Index::new(options, &db_path) + .map_err(|e| IndexError::Error(e))?; + Ok(index) + }).await.expect("thread died"); + println!("after blocking"); + + self.index_store.write().await.insert(meta.uuid.clone(), index?); + + Ok(meta) + } +} + +impl MapIndexStore { + fn new(root: impl AsRef) -> Self { + let mut root = root.as_ref().to_owned(); + root.push("indexes/"); + let meta_store = Arc::new(RwLock::new(HashMap::new())); + let index_store = Arc::new(RwLock::new(HashMap::new())); + Self { meta_store, index_store, root } + } +} diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs new file mode 100644 index 000000000..bfcac7a3f --- /dev/null +++ b/src/index_controller/actor_index_controller/mod.rs @@ -0,0 +1,93 @@ +mod index_actor; +mod update_actor; +mod uuid_resolver; + +use tokio::fs::File; +use tokio::sync::oneshot; +use super::IndexController; +use uuid::Uuid; +use super::IndexMetadata; + + +pub struct ActorIndexController { + uuid_resolver: uuid_resolver::UuidResolverHandle, + index_actor: index_actor::IndexActorHandle, +} + +impl ActorIndexController { + pub fn new() -> Self { + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); + let index_actor = index_actor::IndexActorHandle::new(); + Self { uuid_resolver, index_actor } + } +} + +enum IndexControllerMsg { + CreateIndex { + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + }, + Shutdown, +} + +#[async_trait::async_trait] +impl IndexController for ActorIndexController { + async fn add_documents( + &self, + index: String, + method: milli::update::IndexDocumentsMethod, + format: milli::update::UpdateFormat, + data: File, + primary_key: Option, + ) -> anyhow::Result { + todo!() + } + + fn clear_documents(&self, index: String) -> anyhow::Result { + todo!() + } + + fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { + todo!() + } + + fn update_settings(&self, index_uid: String, settings: super::Settings) -> anyhow::Result { + todo!() + } + + async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { + let super::IndexSettings { name, primary_key } = index_settings; + let uuid = self.uuid_resolver.create(name.unwrap()).await?; + let index_meta = self.index_actor.create_index(uuid, primary_key).await?; + Ok(index_meta) + } + + fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { + todo!() + } + + fn swap_indices(&self, index1_uid: String, index2_uid: String) -> anyhow::Result<()> { + todo!() + } + + fn index(&self, name: String) -> anyhow::Result>> { + todo!() + } + + fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + todo!() + } + + fn all_update_status(&self, index: String) -> anyhow::Result> { + todo!() + } + + fn list_indexes(&self) -> anyhow::Result> { + todo!() + } + + fn update_index(&self, name: String, index_settings: super::IndexSettings) -> anyhow::Result { + todo!() + } +} diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs new file mode 100644 index 000000000..9fd3cc39f --- /dev/null +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -0,0 +1,16 @@ +use super::index_actor::IndexActorHandle; +use uuid::Uuid; +use tokio::sync::{mpsc, oneshot}; + +enum UpdateMsg { + CreateIndex{ + uuid: Uuid, + ret: oneshot::Sender>, + } +} + +struct UpdateActor { + update_store: S, + inbox: mpsc::Receiver, + index_actor: IndexActorHandle, +} diff --git a/src/index_controller/actor_index_controller/uuid_resolver.rs b/src/index_controller/actor_index_controller/uuid_resolver.rs new file mode 100644 index 000000000..d5756d05e --- /dev/null +++ b/src/index_controller/actor_index_controller/uuid_resolver.rs @@ -0,0 +1,116 @@ +use thiserror::Error; +use tokio::sync::{RwLock, mpsc, oneshot}; +use uuid::Uuid; +use std::collections::HashMap; +use std::sync::Arc; +use std::collections::hash_map::Entry; +use log::info; + +pub type Result = std::result::Result; + +#[derive(Debug)] +enum UuidResolveMsg { + Resolve { + name: String, + ret: oneshot::Sender>>, + }, + Create { + name: String, + ret: oneshot::Sender>, + }, + Shutdown, + +} + +struct UuidResolverActor { + inbox: mpsc::Receiver, + store: S, +} + +impl UuidResolverActor { + fn new(inbox: mpsc::Receiver, store: S) -> Self { + Self { inbox, store } + } + + async fn run(mut self) { + use UuidResolveMsg::*; + + info!("uuid resolver started"); + + // TODO: benchmark and use buffered streams to improve throughput. + loop { + match self.inbox.recv().await { + Some(Create { name, ret }) => self.handle_create(name, ret).await, + Some(_) => (), + // all senders have ned dropped, need to quit. + None => break, + } + } + } + + async fn handle_create(&self, name: String, ret: oneshot::Sender>) { + let result = self.store.create_uuid(name).await; + let _ = ret.send(result); + } +} + +#[derive(Clone)] +pub struct UuidResolverHandle { + sender: mpsc::Sender, +} + +impl UuidResolverHandle { + pub fn new() -> Self { + let (sender, reveiver) = mpsc::channel(100); + let store = MapUuidStore(Arc::new(RwLock::new(HashMap::new()))); + let actor = UuidResolverActor::new(reveiver, store); + tokio::spawn(actor.run()); + Self { sender } + } + + pub async fn resolve(&self, name: String) -> anyhow::Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Resolve { name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } + + pub async fn create(&self, name: String) -> anyhow::Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Create { name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } +} + +#[derive(Clone, Debug, Error)] +pub enum UuidError { + #[error("Name already exist.")] + NameAlreadyExist, +} + +#[async_trait::async_trait] +trait UuidStore { + async fn create_uuid(&self, name: String) -> Result; + async fn get_uuid(&self, name: String) -> Result>; +} + +struct MapUuidStore(Arc>>); + +#[async_trait::async_trait] +impl UuidStore for MapUuidStore { + async fn create_uuid(&self, name: String) -> Result { + match self.0.write().await.entry(name) { + Entry::Occupied(_) => Err(UuidError::NameAlreadyExist), + Entry::Vacant(entry) => { + let uuid = Uuid::new_v4(); + let uuid = entry.insert(uuid); + Ok(uuid.clone()) + } + } + } + + async fn get_uuid(&self, name: String) -> Result> { + Ok(self.0.read().await.get(&name).cloned()) + } +} From 6bcc30295028076efe09fc7f78e5daccc09ada1d Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Feb 2021 17:14:11 +0100 Subject: [PATCH 088/522] receive update --- src/data/updates.rs | 22 +- .../actor_index_controller/index_actor.rs | 19 +- .../actor_index_controller/mod.rs | 19 +- .../actor_index_controller/update_actor.rs | 91 +++- .../actor_index_controller/uuid_resolver.rs | 37 +- .../local_index_controller/update_store.rs | 407 ------------------ src/index_controller/mod.rs | 6 +- src/routes/document.rs | 39 +- 8 files changed, 181 insertions(+), 459 deletions(-) delete mode 100644 src/index_controller/local_index_controller/update_store.rs diff --git a/src/data/updates.rs b/src/data/updates.rs index 5cdaf7db1..a559d812a 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,29 +1,33 @@ -use std::ops::Deref; - //use async_compression::tokio_02::write::GzipEncoder; //use futures_util::stream::StreamExt; -//use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; //use tokio::io::AsyncWriteExt; use actix_web::web::Payload; +use tokio::fs::File; +use tokio::io::{AsyncWriteExt, AsyncSeekExt}; +use futures::prelude::stream::StreamExt; use crate::index_controller::UpdateStatus; use crate::index_controller::{Settings, IndexMetadata}; use super::Data; impl Data { - pub async fn add_documents( + pub async fn add_documents( &self, index: impl AsRef + Send + Sync + 'static, method: IndexDocumentsMethod, format: UpdateFormat, - stream: Payload, + mut stream: Payload, primary_key: Option, ) -> anyhow::Result - where - B: Deref, - E: std::error::Error + Send + Sync + 'static, { - let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, stream, primary_key).await?; + let file = tempfile::tempfile_in(".")?; + let mut file = File::from_std(file); + while let Some(Ok(bytes)) = stream.next().await { + file.write(bytes.as_ref()).await; + } + file.seek(std::io::SeekFrom::Start(0)).await?; + let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, file, primary_key).await?; Ok(update_status) } diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 6de0f2b77..40f8de279 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -9,13 +9,15 @@ use milli::Index; use std::collections::hash_map::Entry; use std::fs::create_dir_all; use heed::EnvOpenOptions; -use crate::index_controller::IndexMetadata; +use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; pub type Result = std::result::Result; type AsyncMap = Arc>>; +type UpdateResult = std::result::Result, Failed>; enum IndexMsg { CreateIndex { uuid: Uuid, primary_key: Option, ret: oneshot::Sender> }, + Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, } struct IndexActor { @@ -45,6 +47,7 @@ impl IndexActor { loop { match self.inbox.recv().await { Some(IndexMsg::CreateIndex { uuid, primary_key, ret }) => self.handle_create_index(uuid, primary_key, ret).await, + Some(IndexMsg::Update { ret, meta, data }) => self.handle_update().await, None => break, } } @@ -54,6 +57,10 @@ impl IndexActor { let result = self.store.create_index(uuid, primary_key).await; let _ = ret.send(result); } + + async fn handle_update(&self) { + println!("processing update!!!"); + } } #[derive(Clone)] @@ -77,6 +84,13 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } + + pub async fn update(&self, meta: Processing, data: std::fs::File) -> UpdateResult { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Update { ret, meta, data }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } } struct MapIndexStore { @@ -103,8 +117,6 @@ impl IndexStore for MapIndexStore { let db_path = self.root.join(format!("index-{}", meta.uuid)); - - println!("before blocking"); let index: Result = tokio::task::spawn_blocking(move || { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); @@ -113,7 +125,6 @@ impl IndexStore for MapIndexStore { .map_err(|e| IndexError::Error(e))?; Ok(index) }).await.expect("thread died"); - println!("after blocking"); self.index_store.write().await.insert(meta.uuid.clone(), index?); diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index bfcac7a3f..621bbb1d6 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -1,24 +1,28 @@ mod index_actor; mod update_actor; mod uuid_resolver; +mod update_store; -use tokio::fs::File; use tokio::sync::oneshot; use super::IndexController; use uuid::Uuid; use super::IndexMetadata; +use tokio::fs::File; +use super::UpdateMeta; pub struct ActorIndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, - index_actor: index_actor::IndexActorHandle, + index_handle: index_actor::IndexActorHandle, + update_handle: update_actor::UpdateActorHandle, } impl ActorIndexController { pub fn new() -> Self { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); let index_actor = index_actor::IndexActorHandle::new(); - Self { uuid_resolver, index_actor } + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone()); + Self { uuid_resolver, index_handle: index_actor, update_handle } } } @@ -31,7 +35,7 @@ enum IndexControllerMsg { Shutdown, } -#[async_trait::async_trait] +#[async_trait::async_trait(?Send)] impl IndexController for ActorIndexController { async fn add_documents( &self, @@ -41,7 +45,10 @@ impl IndexController for ActorIndexController { data: File, primary_key: Option, ) -> anyhow::Result { - todo!() + let uuid = self.uuid_resolver.get_or_create(index).await?; + let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; + let status = self.update_handle.update(meta, Some(data), uuid).await?; + Ok(status) } fn clear_documents(&self, index: String) -> anyhow::Result { @@ -59,7 +66,7 @@ impl IndexController for ActorIndexController { async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { let super::IndexSettings { name, primary_key } = index_settings; let uuid = self.uuid_resolver.create(name.unwrap()).await?; - let index_meta = self.index_actor.create_index(uuid, primary_key).await?; + let index_meta = self.index_handle.create_index(uuid, primary_key).await?; Ok(index_meta) } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index 9fd3cc39f..a0bf011c5 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -1,16 +1,99 @@ use super::index_actor::IndexActorHandle; use uuid::Uuid; use tokio::sync::{mpsc, oneshot}; +use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult}; +use thiserror::Error; +use tokio::io::AsyncReadExt; +use log::info; +use tokio::fs::File; +use std::path::PathBuf; +use std::fs::create_dir_all; +use std::sync::Arc; + +pub type Result = std::result::Result; +type UpdateStore = super::update_store::UpdateStore; + +#[derive(Debug, Error)] +pub enum UpdateError {} enum UpdateMsg { CreateIndex{ uuid: Uuid, - ret: oneshot::Sender>, + ret: oneshot::Sender>, + }, + Update { + uuid: Uuid, + meta: UpdateMeta, + payload: Option, + ret: oneshot::Sender> } } -struct UpdateActor { - update_store: S, +struct UpdateActor { + store: Arc, inbox: mpsc::Receiver, - index_actor: IndexActorHandle, + index_handle: IndexActorHandle, +} + +impl UpdateActor { + fn new(store: Arc, inbox: mpsc::Receiver, index_handle: IndexActorHandle) -> Self { + Self { store, inbox, index_handle } + } + + async fn run(mut self) { + + info!("started update actor."); + + loop { + match self.inbox.recv().await { + Some(UpdateMsg::Update { uuid, meta, payload, ret }) => self.handle_update(uuid, meta, payload, ret).await, + Some(_) => {} + None => {} + } + } + } + + async fn handle_update(&self, _uuid: Uuid, meta: UpdateMeta, payload: Option, ret: oneshot::Sender>) { + let mut buf = Vec::new(); + let mut payload = payload.unwrap(); + payload.read_to_end(&mut buf).await.unwrap(); + let result = self.store.register_update(meta, &buf).unwrap(); + let _ = ret.send(Ok(UpdateStatus::Pending(result))); + } +} + +#[derive(Clone)] +pub struct UpdateActorHandle { + sender: mpsc::Sender, +} + +impl UpdateActorHandle { + pub fn new(index_handle: IndexActorHandle) -> Self { + let (sender, receiver) = mpsc::channel(100); + let mut options = heed::EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let mut path = PathBuf::new(); + path.push("data.ms"); + path.push("updates"); + create_dir_all(&path).unwrap(); + let index_handle_clone = index_handle.clone(); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle_clone.update(meta, file)) + }).unwrap(); + let actor = UpdateActor::new(store, receiver, index_handle); + tokio::task::spawn_local(actor.run()); + Self { sender } + } + + pub async fn update(&self, meta: UpdateMeta, payload: Option, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Update { + uuid, + payload, + meta, + ret, + }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } diff --git a/src/index_controller/actor_index_controller/uuid_resolver.rs b/src/index_controller/actor_index_controller/uuid_resolver.rs index d5756d05e..b75c0402c 100644 --- a/src/index_controller/actor_index_controller/uuid_resolver.rs +++ b/src/index_controller/actor_index_controller/uuid_resolver.rs @@ -14,6 +14,10 @@ enum UuidResolveMsg { name: String, ret: oneshot::Sender>>, }, + GetOrCreate { + name: String, + ret: oneshot::Sender>, + }, Create { name: String, ret: oneshot::Sender>, @@ -41,15 +45,21 @@ impl UuidResolverActor { loop { match self.inbox.recv().await { Some(Create { name, ret }) => self.handle_create(name, ret).await, - Some(_) => (), - // all senders have ned dropped, need to quit. + Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, + Some(_) => {} + // all senders have been dropped, need to quit. None => break, } } } async fn handle_create(&self, name: String, ret: oneshot::Sender>) { - let result = self.store.create_uuid(name).await; + let result = self.store.create_uuid(name, true).await; + let _ = ret.send(result); + } + + async fn handle_get_or_create(&self, name: String, ret: oneshot::Sender>) { + let result = self.store.create_uuid(name, false).await; let _ = ret.send(result); } } @@ -75,6 +85,13 @@ impl UuidResolverHandle { Ok(receiver.await.expect("Uuid resolver actor has been killed")?) } + pub async fn get_or_create(&self, name: String) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::GetOrCreate { name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } + pub async fn create(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Create { name, ret }; @@ -91,7 +108,9 @@ pub enum UuidError { #[async_trait::async_trait] trait UuidStore { - async fn create_uuid(&self, name: String) -> Result; + // Create a new entry for `name`. Return an error if `err` and the entry already exists, return + // the uuid otherwise. + async fn create_uuid(&self, name: String, err: bool) -> Result; async fn get_uuid(&self, name: String) -> Result>; } @@ -99,9 +118,15 @@ struct MapUuidStore(Arc>>); #[async_trait::async_trait] impl UuidStore for MapUuidStore { - async fn create_uuid(&self, name: String) -> Result { + async fn create_uuid(&self, name: String, err: bool) -> Result { match self.0.write().await.entry(name) { - Entry::Occupied(_) => Err(UuidError::NameAlreadyExist), + Entry::Occupied(entry) => { + if err { + Err(UuidError::NameAlreadyExist) + } else { + Ok(entry.get().clone()) + } + }, Entry::Vacant(entry) => { let uuid = Uuid::new_v4(); let uuid = entry.insert(uuid); diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs deleted file mode 100644 index b025ff090..000000000 --- a/src/index_controller/local_index_controller/update_store.rs +++ /dev/null @@ -1,407 +0,0 @@ -use std::path::Path; -use std::sync::{Arc, RwLock}; - -use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; -use heed::{EnvOpenOptions, Env, Database}; -use serde::{Serialize, Deserialize}; - -use crate::index_controller::updates::*; - -type BEU64 = heed::zerocopy::U64; - -#[derive(Clone)] -pub struct UpdateStore { - env: Env, - pending_meta: Database, SerdeJson>>, - pending: Database, ByteSlice>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, - processing: Arc>>>, - notification_sender: Sender<()>, -} - -pub trait HandleUpdate { - fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed>; -} - -impl UpdateStore -where - M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, - N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, - E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, -{ - pub fn open( - mut options: EnvOpenOptions, - path: P, - mut update_handler: U, - ) -> heed::Result> - where - P: AsRef, - U: HandleUpdate + Send + 'static, - { - options.max_dbs(5); - - let env = options.open(path)?; - let pending_meta = env.create_database(Some("pending-meta"))?; - let pending = env.create_database(Some("pending"))?; - let processed_meta = env.create_database(Some("processed-meta"))?; - let aborted_meta = env.create_database(Some("aborted-meta"))?; - let failed_meta = env.create_database(Some("failed-meta"))?; - let processing = Arc::new(RwLock::new(None)); - - let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); - // Send a first notification to trigger the process. - let _ = notification_sender.send(()); - - let update_store = Arc::new(UpdateStore { - env, - pending, - pending_meta, - processed_meta, - aborted_meta, - notification_sender, - failed_meta, - processing, - }); - - // We need a weak reference so we can take ownership on the arc later when we - // want to close the index. - let update_store_weak = Arc::downgrade(&update_store); - std::thread::spawn(move || { - // Block and wait for something to process. - 'outer: for _ in notification_receiver { - loop { - match update_store_weak.upgrade() { - Some(update_store) => { - match update_store.process_pending_update(&mut update_handler) { - Ok(Some(_)) => (), - Ok(None) => break, - Err(e) => eprintln!("error while processing update: {}", e), - } - } - // the ownership on the arc has been taken, we need to exit. - None => break 'outer, - } - } - } - }); - - Ok(update_store) - } - - pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { - self.env.prepare_for_closing() - } - - /// Returns the new biggest id to use to store the new update. - fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { - let last_pending = self.pending_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_processed = self.processed_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_aborted = self.aborted_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_update_id = [last_pending, last_processed, last_aborted] - .iter() - .copied() - .flatten() - .max(); - - match last_update_id { - Some(last_id) => Ok(last_id + 1), - None => Ok(0), - } - } - - /// Registers the update content in the pending store and the meta - /// into the pending-meta store. Returns the new unique update id. - pub fn register_update( - &self, - meta: M, - content: &[u8] - ) -> heed::Result> { - let mut wtxn = self.env.write_txn()?; - - // We ask the update store to give us a new update id, this is safe, - // no other update can have the same id because we use a write txn before - // asking for the id and registering it so other update registering - // will be forced to wait for a new write txn. - let update_id = self.new_update_id(&wtxn)?; - let update_key = BEU64::new(update_id); - - let meta = Pending::new(meta, update_id); - self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, content)?; - - wtxn.commit()?; - - if let Err(e) = self.notification_sender.try_send(()) { - assert!(!e.is_disconnected(), "update notification channel is disconnected"); - } - Ok(meta) - } - /// Executes the user provided function on the next pending update (the one with the lowest id). - /// This is asynchronous as it let the user process the update with a read-only txn and - /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, handler: &mut U) -> heed::Result> - where - U: HandleUpdate + Send + 'static, - { - // Create a read transaction to be able to retrieve the pending update in order. - let rtxn = self.env.read_txn()?; - let first_meta = self.pending_meta.first(&rtxn)?; - - // If there is a pending update we process and only keep - // a reader while processing it, not a writer. - match first_meta { - Some((first_id, pending)) => { - let first_content = self.pending - .get(&rtxn, &first_id)? - .expect("associated update content"); - - // we change the state of the update from pending to processing before we pass it - // to the update handler. Processing store is non persistent to be able recover - // from a failure - let processing = pending.processing(); - self.processing - .write() - .unwrap() - .replace(processing.clone()); - // Process the pending update using the provided user function. - let result = handler.handle_update(processing, first_content); - drop(rtxn); - - // Once the pending update have been successfully processed - // we must remove the content from the pending and processing stores and - // write the *new* meta to the processed-meta store and commit. - let mut wtxn = self.env.write_txn()?; - self.processing - .write() - .unwrap() - .take(); - self.pending_meta.delete(&mut wtxn, &first_id)?; - self.pending.delete(&mut wtxn, &first_id)?; - match result { - Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, - Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, - } - wtxn.commit()?; - - Ok(Some(())) - }, - None => Ok(None) - } - } - - /// Execute the user defined function with the meta-store iterators, the first - /// iterator is the *processed* meta one, the second the *aborted* meta one - /// and, the last is the *pending* meta one. - pub fn iter_metas(&self, mut f: F) -> heed::Result - where - F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - ) -> heed::Result, - { - let rtxn = self.env.read_txn()?; - - // We get the pending, processed and aborted meta iterators. - let processed_iter = self.processed_meta.iter(&rtxn)?; - let aborted_iter = self.aborted_meta.iter(&rtxn)?; - let pending_iter = self.pending_meta.iter(&rtxn)?; - let processing = self.processing.read().unwrap().clone(); - let failed_iter = self.failed_meta.iter(&rtxn)?; - - // We execute the user defined function with both iterators. - (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) - } - - /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta(&self, update_id: u64) -> heed::Result>> { - let rtxn = self.env.read_txn()?; - let key = BEU64::new(update_id); - - if let Some(ref meta) = *self.processing.read().unwrap() { - if meta.id() == update_id { - return Ok(Some(UpdateStatus::Processing(meta.clone()))); - } - } - - if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Pending(meta))); - } - - if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Processed(meta))); - } - - if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Aborted(meta))); - } - - if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Failed(meta))); - } - - Ok(None) - } - - /// Aborts an update, an aborted update content is deleted and - /// the meta of it is moved into the aborted updates database. - /// - /// Trying to abort an update that is currently being processed, an update - /// that as already been processed or which doesn't actually exist, will - /// return `None`. - #[allow(dead_code)] - pub fn abort_update(&self, update_id: u64) -> heed::Result>> { - let mut wtxn = self.env.write_txn()?; - let key = BEU64::new(update_id); - - // We cannot abort an update that is currently being processed. - if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { - return Ok(None); - } - - let pending = match self.pending_meta.get(&wtxn, &key)? { - Some(meta) => meta, - None => return Ok(None), - }; - - let aborted = pending.abort(); - - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - - wtxn.commit()?; - - Ok(Some(aborted)) - } - - /// Aborts all the pending updates, and not the one being currently processed. - /// Returns the update metas and ids that were successfully aborted. - #[allow(dead_code)] - pub fn abort_pendings(&self) -> heed::Result)>> { - let mut wtxn = self.env.write_txn()?; - let mut aborted_updates = Vec::new(); - - // We skip the first pending update as it is currently being processed. - for result in self.pending_meta.iter(&wtxn)?.skip(1) { - let (key, pending) = result?; - let id = key.get(); - aborted_updates.push((id, pending.abort())); - } - - for (id, aborted) in &aborted_updates { - let key = BEU64::new(*id); - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - } - - wtxn.commit()?; - - Ok(aborted_updates) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::thread; - use std::time::{Duration, Instant}; - - impl HandleUpdate for F - where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { - fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { - self(meta, content) - } - } - - #[test] - fn simple() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let meta = String::from("kiki"); - let update = update_store.register_update(meta, &[]).unwrap(); - thread::sleep(Duration::from_millis(100)); - let meta = update_store.meta(update.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - } - - #[test] - #[ignore] - fn long_running_update() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let before_register = Instant::now(); - - let meta = String::from("kiki"); - let update_kiki = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("coco"); - let update_coco = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("cucu"); - let update_cucu = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - thread::sleep(Duration::from_millis(400 * 3 + 100)); - - let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "coco processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "cucu processed"); - } else { - panic!() - } - } -} diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index ed085456b..dc6cc3863 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -12,7 +12,7 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; -use actix_web::web::Payload; +use tokio::fs::File; pub use updates::{Processed, Processing, Failed}; @@ -113,7 +113,7 @@ pub struct IndexSettings { /// be provided. This allows the implementer to define the behaviour of write accesses to the /// indices, and abstract the scheduling of the updates. The implementer must be able to provide an /// instance of `IndexStore` -#[async_trait::async_trait] +#[async_trait::async_trait(?Send)] pub trait IndexController { /* @@ -131,7 +131,7 @@ pub trait IndexController { index: String, method: IndexDocumentsMethod, format: UpdateFormat, - data: Payload, + data: File, primary_key: Option, ) -> anyhow::Result; diff --git a/src/routes/document.rs b/src/routes/document.rs index 2cb87074e..00d037359 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -3,7 +3,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use indexmap::IndexMap; use log::error; -//use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; use serde_json::Value; @@ -142,26 +142,25 @@ async fn add_documents_json( params: web::Query, body: Payload, ) -> Result { - todo!() - //let addition_result = data - //.add_documents( - //path.into_inner().index_uid, - //IndexDocumentsMethod::ReplaceDocuments, - //UpdateFormat::Json, - //body, - //params.primary_key.clone(), - //).await; + let addition_result = data + .add_documents( + path.into_inner().index_uid, + IndexDocumentsMethod::ReplaceDocuments, + UpdateFormat::Json, + body, + params.primary_key.clone(), + ).await; - //match addition_result { - //Ok(update) => { - //let value = serde_json::to_string(&update).unwrap(); - //let response = HttpResponse::Ok().body(value); - //Ok(response) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + match addition_result { + Ok(update) => { + let value = serde_json::to_string(&update).unwrap(); + let response = HttpResponse::Ok().body(value); + Ok(response) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } From 658166c05e451b9b53af50e469fa9b194fb3f80e Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Feb 2021 18:11:43 +0100 Subject: [PATCH 089/522] implement document push --- .../actor_index_controller/index_actor.rs | 62 +++-- .../actor_index_controller/mod.rs | 2 +- .../actor_index_controller/update_actor.rs | 4 +- .../local_index_controller/update_handler.rs | 255 ------------------ src/index_controller/updates.rs | 9 +- 5 files changed, 58 insertions(+), 274 deletions(-) delete mode 100644 src/index_controller/local_index_controller/update_handler.rs diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 40f8de279..6123ca774 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -1,15 +1,20 @@ -use uuid::Uuid; +use std::fs::{File, create_dir_all}; use std::path::{PathBuf, Path}; -use chrono::Utc; -use tokio::sync::{mpsc, oneshot, RwLock}; -use thiserror::Error; -use std::collections::HashMap; use std::sync::Arc; -use milli::Index; -use std::collections::hash_map::Entry; -use std::fs::create_dir_all; + +use chrono::Utc; use heed::EnvOpenOptions; +use milli::Index; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot, RwLock}; +use uuid::Uuid; +use log::info; + +use super::update_handler::UpdateHandler; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; +use crate::option::IndexerOpts; pub type Result = std::result::Result; type AsyncMap = Arc>>; @@ -22,6 +27,7 @@ enum IndexMsg { struct IndexActor { inbox: mpsc::Receiver, + update_handler: Arc, store: S, } @@ -36,18 +42,22 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn get_or_create(&self, uuid: Uuid) -> Result>; } -impl IndexActor { +impl IndexActor { fn new(inbox: mpsc::Receiver, store: S) -> Self { - Self { inbox, store } + let options = IndexerOpts::default(); + let update_handler = UpdateHandler::new(&options).unwrap(); + let update_handler = Arc::new(update_handler); + Self { inbox, store, update_handler } } async fn run(mut self) { loop { match self.inbox.recv().await { Some(IndexMsg::CreateIndex { uuid, primary_key, ret }) => self.handle_create_index(uuid, primary_key, ret).await, - Some(IndexMsg::Update { ret, meta, data }) => self.handle_update().await, + Some(IndexMsg::Update { ret, meta, data }) => self.handle_update(meta, data, ret).await, None => break, } } @@ -58,8 +68,14 @@ impl IndexActor { let _ = ret.send(result); } - async fn handle_update(&self) { - println!("processing update!!!"); + async fn handle_update(&self, meta: Processing, data: File, ret: oneshot::Sender) { + info!("processing update"); + let uuid = meta.index_uuid().clone(); + let index = self.store.get_or_create(uuid).await.unwrap(); + let update_handler = self.update_handler.clone(); + let result = tokio::task::spawn_blocking(move || update_handler.handle_update(meta, data, index.as_ref())).await; + let result = result.unwrap(); + let _ = ret.send(result); } } @@ -96,7 +112,7 @@ impl IndexActorHandle { struct MapIndexStore { root: PathBuf, meta_store: AsyncMap, - index_store: AsyncMap, + index_store: AsyncMap>, } #[async_trait::async_trait] @@ -126,10 +142,26 @@ impl IndexStore for MapIndexStore { Ok(index) }).await.expect("thread died"); - self.index_store.write().await.insert(meta.uuid.clone(), index?); + self.index_store.write().await.insert(meta.uuid.clone(), Arc::new(index?)); Ok(meta) } + + async fn get_or_create(&self, uuid: Uuid) -> Result> { + match self.index_store.write().await.entry(uuid.clone()) { + Entry::Vacant(entry) => { + match self.meta_store.write().await.entry(uuid.clone()) { + Entry::Vacant(_) => { + todo!() + } + Entry::Occupied(entry) => { + todo!() + } + } + } + Entry::Occupied(entry) => Ok(entry.get().clone()), + } + } } impl MapIndexStore { diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 621bbb1d6..2936c59ea 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -2,6 +2,7 @@ mod index_actor; mod update_actor; mod uuid_resolver; mod update_store; +mod update_handler; use tokio::sync::oneshot; use super::IndexController; @@ -10,7 +11,6 @@ use super::IndexMetadata; use tokio::fs::File; use super::UpdateMeta; - pub struct ActorIndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index a0bf011c5..d182ef1c8 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -53,11 +53,11 @@ impl UpdateActor { } } - async fn handle_update(&self, _uuid: Uuid, meta: UpdateMeta, payload: Option, ret: oneshot::Sender>) { + async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, payload: Option, ret: oneshot::Sender>) { let mut buf = Vec::new(); let mut payload = payload.unwrap(); payload.read_to_end(&mut buf).await.unwrap(); - let result = self.store.register_update(meta, &buf).unwrap(); + let result = self.store.register_update(meta, &buf, uuid).unwrap(); let _ = ret.send(Ok(UpdateStatus::Pending(result))); } } diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs deleted file mode 100644 index 5781a2806..000000000 --- a/src/index_controller/local_index_controller/update_handler.rs +++ /dev/null @@ -1,255 +0,0 @@ -use std::collections::HashMap; -use std::io; -use std::sync::Arc; - -use anyhow::Result; -use flate2::read::GzDecoder; -use grenad::CompressionType; -use log::info; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use milli::Index; -use rayon::ThreadPool; - -use super::update_store::HandleUpdate; -use crate::index_controller::updates::{Failed, Processed, Processing}; -use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; -use crate::option::IndexerOpts; - -pub struct UpdateHandler { - index: Arc, - max_nb_chunks: Option, - chunk_compression_level: Option, - thread_pool: Arc, - log_frequency: usize, - max_memory: usize, - linked_hash_map_size: usize, - chunk_compression_type: CompressionType, - chunk_fusing_shrink_size: u64, -} - -impl UpdateHandler { - pub fn new( - opt: &IndexerOpts, - index: Arc, - thread_pool: Arc, - ) -> anyhow::Result { - Ok(Self { - index, - max_nb_chunks: opt.max_nb_chunks, - chunk_compression_level: opt.chunk_compression_level, - thread_pool, - log_frequency: opt.log_every_n, - max_memory: opt.max_memory.get_bytes() as usize, - linked_hash_map_size: opt.linked_hash_map_size, - chunk_compression_type: opt.chunk_compression_type, - chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), - }) - } - - fn update_buidler(&self, update_id: u64) -> UpdateBuilder { - // We prepare the update by using the update builder. - let mut update_builder = UpdateBuilder::new(update_id); - if let Some(max_nb_chunks) = self.max_nb_chunks { - update_builder.max_nb_chunks(max_nb_chunks); - } - if let Some(chunk_compression_level) = self.chunk_compression_level { - update_builder.chunk_compression_level(chunk_compression_level); - } - update_builder.thread_pool(&self.thread_pool); - update_builder.log_every_n(self.log_frequency); - update_builder.max_memory(self.max_memory); - update_builder.linked_hash_map_size(self.linked_hash_map_size); - update_builder.chunk_compression_type(self.chunk_compression_type); - update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); - update_builder - } - - fn update_documents( - &self, - format: UpdateFormat, - method: IndexDocumentsMethod, - content: &[u8], - update_builder: UpdateBuilder, - primary_key: Option<&str>, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.index.write_txn()?; - - // Set the primary key if not set already, ignore if already set. - match (self.index.primary_key(&wtxn)?, primary_key) { - (None, Some(ref primary_key)) => { - self.index.put_primary_key(&mut wtxn, primary_key)?; - } - _ => (), - } - - let mut builder = update_builder.index_documents(&mut wtxn, &self.index); - builder.update_format(format); - builder.index_documents_method(method); - - let gzipped = true; - let reader = if gzipped && !content.is_empty() { - Box::new(GzDecoder::new(content)) - } else { - Box::new(content) as Box - }; - - let result = builder.execute(reader, |indexing_step, update_id| { - info!("update {}: {:?}", update_id, indexing_step) - }); - - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.index.write_txn()?; - let builder = update_builder.clear_documents(&mut wtxn, &self.index); - - match builder.execute() { - Ok(_count) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_settings( - &self, - settings: &Settings, - update_builder: UpdateBuilder, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.index.write_txn()?; - let mut builder = update_builder.settings(&mut wtxn, &self.index); - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.searchable_attributes { - match names { - Some(names) => builder.set_searchable_fields(names.clone()), - None => builder.reset_searchable_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.displayed_attributes { - match names { - Some(names) => builder.set_displayed_fields(names.clone()), - None => builder.reset_displayed_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); - builder.set_faceted_fields(facet_types); - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { - match criteria { - Some(criteria) => builder.set_criteria(criteria.clone()), - None => builder.reset_criteria(), - } - } - - let result = builder - .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_facets( - &self, - levels: &Facets, - update_builder: UpdateBuilder, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.index.write_txn()?; - let mut builder = update_builder.facets(&mut wtxn, &self.index); - if let Some(value) = levels.level_group_size { - builder.level_group_size(value); - } - if let Some(value) = levels.min_level_size { - builder.min_level_size(value); - } - match builder.execute() { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn delete_documents( - &self, - document_ids: &[u8], - update_builder: UpdateBuilder, - ) -> anyhow::Result { - let ids: Vec = serde_json::from_slice(document_ids)?; - let mut txn = self.index.write_txn()?; - let mut builder = update_builder.delete_documents(&mut txn, &self.index)?; - - // We ignore unexisting document ids - ids.iter().for_each(|id| { builder.delete_external_id(id); }); - - match builder.execute() { - Ok(deleted) => txn - .commit() - .and(Ok(UpdateResult::DocumentDeletion { deleted })) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } -} - -impl HandleUpdate for UpdateHandler { - fn handle_update( - &mut self, - meta: Processing, - content: &[u8], - ) -> Result, Failed> { - use UpdateMeta::*; - - let update_id = meta.id(); - - let update_builder = self.update_buidler(update_id); - - let result = match meta.meta() { - DocumentsAddition { - method, - format, - primary_key, - } => self.update_documents( - *format, - *method, - content, - update_builder, - primary_key.as_deref(), - ), - ClearDocuments => self.clear_documents(update_builder), - DeleteDocuments => self.delete_documents(content, update_builder), - Settings(settings) => self.update_settings(settings, update_builder), - Facets(levels) => self.update_facets(levels, update_builder), - }; - - match result { - Ok(result) => Ok(meta.process(result)), - Err(e) => Err(meta.fail(e.to_string())), - } - } -} diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs index b15593b58..b2ad54a14 100644 --- a/src/index_controller/updates.rs +++ b/src/index_controller/updates.rs @@ -1,5 +1,6 @@ use chrono::{Utc, DateTime}; use serde::{Serialize, Deserialize}; +use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -7,14 +8,16 @@ pub struct Pending { pub update_id: u64, pub meta: M, pub enqueued_at: DateTime, + pub index_uuid: Uuid, } impl Pending { - pub fn new(meta: M, update_id: u64) -> Self { + pub fn new(meta: M, update_id: u64, index_uuid: Uuid) -> Self { Self { enqueued_at: Utc::now(), meta, update_id, + index_uuid, } } @@ -73,6 +76,10 @@ impl Processing { self.from.meta() } + pub fn index_uuid(&self) -> &Uuid { + &self.from.index_uuid + } + pub fn process(self, meta: N) -> Processed { Processed { success: meta, From c994fe4609957a4865bcd3fe184765476c08c9c4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sun, 28 Feb 2021 10:08:36 +0100 Subject: [PATCH 090/522] add license --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..87471adb1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Meili + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 5ba58c1e9c6c1995fb46b4d0ecba2b118ab862d9 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sun, 28 Feb 2021 10:09:56 +0100 Subject: [PATCH 091/522] add Marin to authors --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 993c80946..4a00667b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -authors = ["Quentin de Quelen ", "Clément Renault "] +authors = ["Quentin de Quelen ", "Clément Renault ", "Marin Postma "] description = "MeiliSearch HTTP server" edition = "2018" license = "MIT" From 79708aeb67b4415879468cc7c4d119ed0f1c986c Mon Sep 17 00:00:00 2001 From: mpostma Date: Sun, 28 Feb 2021 16:33:02 +0100 Subject: [PATCH 092/522] add milli as git dep --- Cargo.lock | 152 +++++++++++++++++++++++++++++++++++++---------------- Cargo.toml | 4 +- 2 files changed, 110 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e4054d87..ec936a450 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -325,9 +325,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "assert-json-diff" @@ -513,9 +513,9 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" dependencies = [ "lazy_static", "memchr", @@ -545,10 +545,16 @@ dependencies = [ ] [[package]] -name = "byteorder" -version = "1.3.4" +name = "bytemuck" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "bytes" @@ -920,15 +926,15 @@ checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "winapi 0.3.9", ] [[package]] name = "flate2" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" dependencies = [ "cfg-if 1.0.0", "crc32fast", @@ -1543,9 +1549,9 @@ checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" [[package]] name = "linked-hash-map" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lmdb-rkv-sys" @@ -1569,11 +1575,11 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -1717,6 +1723,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" +source = "git+https://github.com/meilisearch/milli.git?rev=8dcb3e0#8dcb3e0c41965c96ae718ae85c45004cf94c6e94" dependencies = [ "anyhow", "bstr", @@ -1730,7 +1737,7 @@ dependencies = [ "grenad", "heed", "human_format", - "itertools 0.9.0", + "itertools 0.10.0", "levenshtein_automata", "linked-hash-map", "log", @@ -1927,9 +1934,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "ordered-float" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dacdec97876ef3ede8c50efc429220641a0b11ba0048b4b0c357bccbc47c5204" +checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" dependencies = [ "num-traits", ] @@ -1964,7 +1971,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "smallvec", "winapi 0.3.9", ] @@ -2215,12 +2222,24 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom 0.1.15", "libc", - "rand_chacha", + "rand_chacha 0.2.2", "rand_core 0.5.1", - "rand_hc", + "rand_hc 0.2.0", "rand_pcg", ] +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -2231,6 +2250,16 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -2255,6 +2284,15 @@ dependencies = [ "getrandom 0.1.15", ] +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -2264,6 +2302,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + [[package]] name = "rand_pcg" version = "0.2.1" @@ -2323,10 +2370,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] -name = "regex" -version = "1.4.2" +name = "redox_syscall" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "aho-corasick", "memchr", @@ -2345,9 +2401,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "remove_dir_all" @@ -2405,6 +2461,12 @@ dependencies = [ "quick-error", ] +[[package]] +name = "retain_mut" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" + [[package]] name = "ring" version = "0.16.19" @@ -2422,11 +2484,13 @@ dependencies = [ [[package]] name = "roaring" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d60b41c8f25d07cecab125cb46ebbf234fc055effc61ca2392a3ef4f9422304" +checksum = "c6744a4a918e91359ad1d356a91e2e943a86d9fb9ae77f715d617032ea2af88f" dependencies = [ + "bytemuck", "byteorder", + "retain_mut", ] [[package]] @@ -2535,18 +2599,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.118" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.118" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" dependencies = [ "proc-macro2", "quote", @@ -2555,9 +2619,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.60" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "indexmap", "itoa", @@ -2680,9 +2744,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" @@ -2797,9 +2861,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.55" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a571a711dddd09019ccc628e1b17fe87c59b09d513c06c026877aa708334f37a" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" dependencies = [ "proc-macro2", "quote", @@ -2835,7 +2899,7 @@ checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" dependencies = [ "filetime", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "xattr", ] @@ -2851,14 +2915,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "rand 0.7.3", - "redox_syscall", + "rand 0.8.3", + "redox_syscall 0.2.5", "remove_dir_all", "winapi 0.3.9", ] diff --git a/Cargo.toml b/Cargo.toml index 4a00667b1..875165588 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ actix-http = "2" actix-rt = "1" actix-service = "1.0.6" actix-web = { version = "3.3.2", features = ["rustls"] } -anyhow = "1.0.36" +anyhow = "1.0.38" async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } bytes = "0.6.0" @@ -38,7 +38,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { path = "../milli/milli" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "8dcb3e0" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" From a9a9ed6318696fa2a5199bdee3c2489b044aa143 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sun, 28 Feb 2021 16:41:47 +0100 Subject: [PATCH 093/522] create workspace with meilisearch-error --- .gitignore | 8 +- Cargo.lock | 395 +- Cargo.toml | 85 +- meilisearch-error/Cargo.toml | 8 + meilisearch-error/src/lib.rs | 185 + meilisearch-http/Cargo.lock | 3581 +++++++++++++++++ meilisearch-http/Cargo.toml | 79 + meilisearch-http/_tests/dashboard.rs | 12 + meilisearch-http/_tests/documents_add.rs | 222 + meilisearch-http/_tests/documents_delete.rs | 67 + meilisearch-http/_tests/documents_get.rs | 23 + meilisearch-http/_tests/dump.rs | 395 ++ meilisearch-http/_tests/errors.rs | 200 + meilisearch-http/_tests/health.rs | 11 + meilisearch-http/_tests/index.rs | 809 ++++ meilisearch-http/_tests/index_update.rs | 200 + .../_tests/lazy_index_creation.rs | 446 ++ meilisearch-http/_tests/placeholder_search.rs | 629 +++ meilisearch-http/_tests/search.rs | 1879 +++++++++ meilisearch-http/_tests/search_settings.rs | 538 +++ meilisearch-http/_tests/settings.rs | 523 +++ .../_tests/settings_ranking_rules.rs | 182 + .../_tests/settings_stop_words.rs | 61 + meilisearch-http/_tests/url_normalizer.rs | 18 + build.rs => meilisearch-http/build.rs | 0 .../data.mdb | Bin 0 -> 32768 bytes .../lock.mdb | Bin 0 -> 8192 bytes .../data.mdb | Bin 0 -> 45056 bytes .../lock.mdb | Bin 0 -> 8192 bytes meilisearch-http/data.ms/updates/data.mdb | Bin 0 -> 65536 bytes meilisearch-http/data.ms/updates/lock.mdb | Bin 0 -> 8192 bytes .../public}/bulma.min.css | 0 .../public}/interface.html | 0 {src => meilisearch-http/src}/analytics.rs | 0 {src => meilisearch-http/src}/data/mod.rs | 0 {src => meilisearch-http/src}/data/search.rs | 0 {src => meilisearch-http/src}/data/updates.rs | 0 {src => meilisearch-http/src}/dump.rs | 0 {src => meilisearch-http/src}/error.rs | 0 .../src}/helpers/authentication.rs | 0 .../src}/helpers/compression.rs | 0 {src => meilisearch-http/src}/helpers/mod.rs | 0 .../src}/helpers/normalize_path.rs | 0 .../actor_index_controller/update_handler.rs | 260 ++ .../actor_index_controller/update_store.rs | 423 ++ .../local_index_controller/index_store.rs | 0 .../local_index_controller/mod.rs | 0 .../local_index_controller/update_handler.rs | 0 .../local_index_controller/update_store.rs | 0 .../src}/index_controller/mod.rs | 0 .../src}/index_controller/updates.rs | 0 {src => meilisearch-http/src}/lib.rs | 0 {src => meilisearch-http/src}/main.rs | 0 {src => meilisearch-http/src}/option.rs | 0 .../src}/routes/document.rs | 0 {src => meilisearch-http/src}/routes/dump.rs | 0 .../src}/routes/health.rs | 0 {src => meilisearch-http/src}/routes/index.rs | 0 {src => meilisearch-http/src}/routes/key.rs | 0 {src => meilisearch-http/src}/routes/mod.rs | 0 .../src}/routes/search.rs | 0 .../settings/attributes_for_faceting.rs | 0 .../routes/settings/displayed_attributes.rs | 0 .../routes/settings/distinct_attributes.rs | 0 .../src}/routes/settings/mod.rs | 0 .../src}/routes/settings/ranking_rules.rs | 0 .../routes/settings/searchable_attributes.rs | 0 .../src}/routes/settings/stop_words.rs | 0 .../src}/routes/settings/synonyms.rs | 0 {src => meilisearch-http/src}/routes/stats.rs | 0 .../src}/routes/stop_words.rs | 0 .../src}/routes/synonym.rs | 0 {src => meilisearch-http/src}/snapshot.rs | 0 .../tests/assets/dumps/v1/metadata.json | 12 + .../assets/dumps/v1/test/documents.jsonl | 77 + .../tests/assets/dumps/v1/test/settings.json | 59 + .../tests/assets/dumps/v1/test/updates.jsonl | 2 + meilisearch-http/tests/assets/test_set.json | 1613 ++++++++ .../tests}/common/index.rs | 0 .../tests}/common/mod.rs | 0 .../tests}/common/server.rs | 0 .../tests}/common/service.rs | 0 .../tests}/documents/add_documents.rs | 0 .../tests}/documents/delete_documents.rs | 0 .../tests}/documents/get_documents.rs | 0 .../tests}/documents/mod.rs | 0 .../tests}/index/create_index.rs | 0 .../tests}/index/delete_index.rs | 0 .../tests}/index/get_index.rs | 0 .../tests}/index/mod.rs | 0 .../tests}/index/update_index.rs | 0 .../tests}/integration.rs | 0 .../tests}/search/mod.rs | 0 .../tests}/settings/get_settings.rs | 0 .../tests}/settings/mod.rs | 0 .../tests}/updates/mod.rs | 0 96 files changed, 12737 insertions(+), 265 deletions(-) create mode 100644 meilisearch-error/Cargo.toml create mode 100644 meilisearch-error/src/lib.rs create mode 100644 meilisearch-http/Cargo.lock create mode 100644 meilisearch-http/Cargo.toml create mode 100644 meilisearch-http/_tests/dashboard.rs create mode 100644 meilisearch-http/_tests/documents_add.rs create mode 100644 meilisearch-http/_tests/documents_delete.rs create mode 100644 meilisearch-http/_tests/documents_get.rs create mode 100644 meilisearch-http/_tests/dump.rs create mode 100644 meilisearch-http/_tests/errors.rs create mode 100644 meilisearch-http/_tests/health.rs create mode 100644 meilisearch-http/_tests/index.rs create mode 100644 meilisearch-http/_tests/index_update.rs create mode 100644 meilisearch-http/_tests/lazy_index_creation.rs create mode 100644 meilisearch-http/_tests/placeholder_search.rs create mode 100644 meilisearch-http/_tests/search.rs create mode 100644 meilisearch-http/_tests/search_settings.rs create mode 100644 meilisearch-http/_tests/settings.rs create mode 100644 meilisearch-http/_tests/settings_ranking_rules.rs create mode 100644 meilisearch-http/_tests/settings_stop_words.rs create mode 100644 meilisearch-http/_tests/url_normalizer.rs rename build.rs => meilisearch-http/build.rs (100%) create mode 100644 meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/data.mdb create mode 100644 meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/lock.mdb create mode 100644 meilisearch-http/data.ms/indexes/index-eb9d0392-4386-4fa1-b68c-dac0d93bb16e/data.mdb create mode 100644 meilisearch-http/data.ms/indexes/index-eb9d0392-4386-4fa1-b68c-dac0d93bb16e/lock.mdb create mode 100644 meilisearch-http/data.ms/updates/data.mdb create mode 100644 meilisearch-http/data.ms/updates/lock.mdb rename {public => meilisearch-http/public}/bulma.min.css (100%) rename {public => meilisearch-http/public}/interface.html (100%) rename {src => meilisearch-http/src}/analytics.rs (100%) rename {src => meilisearch-http/src}/data/mod.rs (100%) rename {src => meilisearch-http/src}/data/search.rs (100%) rename {src => meilisearch-http/src}/data/updates.rs (100%) rename {src => meilisearch-http/src}/dump.rs (100%) rename {src => meilisearch-http/src}/error.rs (100%) rename {src => meilisearch-http/src}/helpers/authentication.rs (100%) rename {src => meilisearch-http/src}/helpers/compression.rs (100%) rename {src => meilisearch-http/src}/helpers/mod.rs (100%) rename {src => meilisearch-http/src}/helpers/normalize_path.rs (100%) create mode 100644 meilisearch-http/src/index_controller/actor_index_controller/update_handler.rs create mode 100644 meilisearch-http/src/index_controller/actor_index_controller/update_store.rs rename {src => meilisearch-http/src}/index_controller/local_index_controller/index_store.rs (100%) rename {src => meilisearch-http/src}/index_controller/local_index_controller/mod.rs (100%) rename {src => meilisearch-http/src}/index_controller/local_index_controller/update_handler.rs (100%) rename {src => meilisearch-http/src}/index_controller/local_index_controller/update_store.rs (100%) rename {src => meilisearch-http/src}/index_controller/mod.rs (100%) rename {src => meilisearch-http/src}/index_controller/updates.rs (100%) rename {src => meilisearch-http/src}/lib.rs (100%) rename {src => meilisearch-http/src}/main.rs (100%) rename {src => meilisearch-http/src}/option.rs (100%) rename {src => meilisearch-http/src}/routes/document.rs (100%) rename {src => meilisearch-http/src}/routes/dump.rs (100%) rename {src => meilisearch-http/src}/routes/health.rs (100%) rename {src => meilisearch-http/src}/routes/index.rs (100%) rename {src => meilisearch-http/src}/routes/key.rs (100%) rename {src => meilisearch-http/src}/routes/mod.rs (100%) rename {src => meilisearch-http/src}/routes/search.rs (100%) rename {src => meilisearch-http/src}/routes/settings/attributes_for_faceting.rs (100%) rename {src => meilisearch-http/src}/routes/settings/displayed_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/distinct_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/mod.rs (100%) rename {src => meilisearch-http/src}/routes/settings/ranking_rules.rs (100%) rename {src => meilisearch-http/src}/routes/settings/searchable_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/stop_words.rs (100%) rename {src => meilisearch-http/src}/routes/settings/synonyms.rs (100%) rename {src => meilisearch-http/src}/routes/stats.rs (100%) rename {src => meilisearch-http/src}/routes/stop_words.rs (100%) rename {src => meilisearch-http/src}/routes/synonym.rs (100%) rename {src => meilisearch-http/src}/snapshot.rs (100%) create mode 100644 meilisearch-http/tests/assets/dumps/v1/metadata.json create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/settings.json create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl create mode 100644 meilisearch-http/tests/assets/test_set.json rename {tests => meilisearch-http/tests}/common/index.rs (100%) rename {tests => meilisearch-http/tests}/common/mod.rs (100%) rename {tests => meilisearch-http/tests}/common/server.rs (100%) rename {tests => meilisearch-http/tests}/common/service.rs (100%) rename {tests => meilisearch-http/tests}/documents/add_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/delete_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/get_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/mod.rs (100%) rename {tests => meilisearch-http/tests}/index/create_index.rs (100%) rename {tests => meilisearch-http/tests}/index/delete_index.rs (100%) rename {tests => meilisearch-http/tests}/index/get_index.rs (100%) rename {tests => meilisearch-http/tests}/index/mod.rs (100%) rename {tests => meilisearch-http/tests}/index/update_index.rs (100%) rename {tests => meilisearch-http/tests}/integration.rs (100%) rename {tests => meilisearch-http/tests}/search/mod.rs (100%) rename {tests => meilisearch-http/tests}/settings/get_settings.rs (100%) rename {tests => meilisearch-http/tests}/settings/mod.rs (100%) rename {tests => meilisearch-http/tests}/updates/mod.rs (100%) diff --git a/.gitignore b/.gitignore index 1d71f78ca..e1f56a99c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ -# the milli project is a library /target +meilisearch-core/target +**/*.csv +**/*.json_lines +**/*.rs.bk +/*.mdb +/query-history.txt +/data.ms diff --git a/Cargo.lock b/Cargo.lock index ec936a450..9af2e7d12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,9 +40,9 @@ dependencies = [ [[package]] name = "actix-cors" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3a3d5493dbc9b8769fe88c030d057ef8d2edc5728e5e26267780e8fc5db0be" +checksum = "36b133d8026a9f209a9aeeeacd028e7451bcca975f592881b305d37983f303d7" dependencies = [ "actix-web", "derive_more", @@ -89,15 +89,15 @@ dependencies = [ "log", "mime", "percent-encoding", - "pin-project 1.0.2", + "pin-project 1.0.5", "rand 0.7.3", "regex", "serde", "serde_json", "serde_urlencoded", - "sha-1 0.9.2", + "sha-1 0.9.4", "slab", - "time 0.2.23", + "time 0.2.25", ] [[package]] @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd1f7dbda1645bf7da33554db60891755f6c01c1b2169e2f4c492098d30c235" +checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" dependencies = [ "bytestring", "http", @@ -261,14 +261,14 @@ dependencies = [ "fxhash", "log", "mime", - "pin-project 1.0.2", + "pin-project 1.0.5", "regex", "rustls", "serde", "serde_json", "serde_urlencoded", "socket2", - "time 0.2.23", + "time 0.2.25", "tinyvec", "url", ] @@ -286,18 +286,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" dependencies = [ "gimli", ] [[package]] name = "adler" -version = "0.2.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" @@ -332,7 +332,7 @@ checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "assert-json-diff" version = "1.0.1" -source = "git+https://github.com/qdequele/assert-json-diff#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" dependencies = [ "serde", "serde_json", @@ -340,14 +340,14 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a" +checksum = "b72c1f1154e234325b50864a349b9c8e56939e266a4c307c0f159812df2f9537" dependencies = [ "flate2", "futures-core", "memchr", - "pin-project-lite 0.1.11", + "pin-project-lite 0.2.4", "tokio", ] @@ -406,9 +406,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.55" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -525,9 +525,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.4.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" [[package]] name = "byte-tools" @@ -569,19 +569,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" [[package]] -name = "bytestring" -version = "0.1.5" +name = "bytes" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7c05fa5172da78a62d9949d662d2ac89d4cc7355d7b49adee5163f1fb3f363" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "bytestring" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", ] [[package]] name = "cc" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" dependencies = [ "jobserver", ] @@ -647,9 +653,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" [[package]] name = "cookie" @@ -658,7 +664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" dependencies = [ "percent-encoding", - "time 0.2.23", + "time 0.2.25", "version_check", ] @@ -696,7 +702,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", ] [[package]] @@ -707,18 +713,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", ] [[package]] name = "crossbeam-epoch" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" dependencies = [ "cfg-if 1.0.0", - "const_fn", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", "lazy_static", "memoffset", "scopeguard", @@ -745,9 +750,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -845,9 +850,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.26" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" dependencies = [ "cfg-if 1.0.0", ] @@ -879,12 +884,12 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" dependencies = [ "atty", - "humantime 2.0.1", + "humantime 2.1.0", "log", "regex", "termcolor", @@ -920,13 +925,13 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "filetime" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "winapi 0.3.9", ] @@ -950,9 +955,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", "percent-encoding", @@ -994,9 +999,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" dependencies = [ "futures-channel", "futures-core", @@ -1009,9 +1014,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" dependencies = [ "futures-core", "futures-sink", @@ -1019,15 +1024,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" +checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" [[package]] name = "futures-executor" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" dependencies = [ "futures-core", "futures-task", @@ -1036,15 +1041,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" +checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" [[package]] name = "futures-macro" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -1054,24 +1059,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" +checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" [[package]] name = "futures-task" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" -dependencies = [ - "once_cell", -] +checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" [[package]] name = "futures-util" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" dependencies = [ "futures-channel", "futures-core", @@ -1080,7 +1082,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.2", + "pin-project-lite 0.2.4", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -1117,11 +1119,11 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1248,9 +1250,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] @@ -1268,11 +1270,11 @@ dependencies = [ [[package]] name = "http" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84129d298a6d57d246960ff8eb831ca4af3f96d29e2e28848dae275408658e26" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "fnv", "itoa", ] @@ -1289,9 +1291,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.4" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" [[package]] name = "httpdate" @@ -1316,15 +1318,15 @@ dependencies = [ [[package]] name = "humantime" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.13.9" +version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" dependencies = [ "bytes 0.5.6", "futures-channel", @@ -1336,7 +1338,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.2", + "pin-project 1.0.5", "socket2", "tokio", "tower-service", @@ -1362,9 +1364,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" dependencies = [ "matches", "unicode-bidi", @@ -1452,9 +1454,9 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jemalloc-sys" @@ -1503,9 +1505,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.46" +version = "0.3.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" dependencies = [ "wasm-bindgen", ] @@ -1543,9 +1545,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" [[package]] name = "linked-hash-map" @@ -1640,7 +1642,7 @@ dependencies = [ "crossbeam-channel", "dashmap", "either", - "env_logger 0.8.2", + "env_logger 0.8.3", "flate2", "fst", "futures", @@ -1682,7 +1684,7 @@ dependencies = [ [[package]] name = "meilisearch-tokenizer" version = "0.1.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#147b6154b1b34cb8f5da2df6a416b7da191bc850" +source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#31ba3ff4a15501f12b7d37ac64ddce7c35a9757c" dependencies = [ "character_converter", "cow-utils", @@ -1778,9 +1780,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", "autocfg", @@ -1904,9 +1906,9 @@ dependencies = [ [[package]] name = "object" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" [[package]] name = "obkv" @@ -1916,9 +1918,9 @@ checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" [[package]] name = "once_cell" -version = "1.5.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" +checksum = "10acf907b94fc1b1a152d08ef97e7759650268cf986bf127f387e602b02c7e5a" [[package]] name = "opaque-debug" @@ -1964,14 +1966,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "smallvec", "winapi 0.3.9", ] @@ -2082,11 +2084,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" dependencies = [ - "pin-project-internal 1.0.2", + "pin-project-internal 1.0.5", ] [[package]] @@ -2102,9 +2104,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ "proc-macro2", "quote", @@ -2119,9 +2121,9 @@ checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" [[package]] name = "pin-project-lite" -version = "0.2.0" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" [[package]] name = "pin-utils" @@ -2173,9 +2175,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro-nested" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" @@ -2194,9 +2196,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] @@ -2220,7 +2222,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -2281,7 +2283,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", ] [[package]] @@ -2349,7 +2351,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", "lazy_static", "num_cpus", ] @@ -2363,12 +2365,6 @@ dependencies = [ "rand_core 0.3.1", ] -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.2.5" @@ -2436,7 +2432,7 @@ dependencies = [ "mime", "mime_guess", "percent-encoding", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.4", "rustls", "serde", "serde_json", @@ -2469,9 +2465,9 @@ checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" [[package]] name = "ring" -version = "0.16.19" +version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ "cc", "libc", @@ -2505,7 +2501,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", ] [[package]] @@ -2549,7 +2554,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", ] [[package]] @@ -2558,6 +2572,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "sentry" version = "0.18.1" @@ -2576,7 +2599,7 @@ dependencies = [ "rand 0.7.3", "regex", "reqwest", - "rustc_version", + "rustc_version 0.2.3", "sentry-types", "uname", "url", @@ -2631,9 +2654,9 @@ dependencies = [ [[package]] name = "serde_url_params" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +checksum = "2c43307d0640738af32fe8d01e47119bc0fc8a686be470a44a586caff76dfb34" dependencies = [ "serde", "url", @@ -2665,9 +2688,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -2684,9 +2707,9 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -2750,9 +2773,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98d3306e84bf86710d6cd8b4c9c3b721d5454cc91a603180f8f8cd06cfd317b4" +checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a" [[package]] name = "socket2" @@ -2773,9 +2796,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.13" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" +checksum = "a2beb4d1860a61f571530b3f855a1b538d0200f7871c63331ecd6f17b1f014f8" dependencies = [ "version_check", ] @@ -2787,7 +2810,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -2893,13 +2916,12 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.30" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +checksum = "c0bcfbd6a598361fda270d82469fff3d65089dc33e175c9a131f7b4cd395f228" dependencies = [ "filetime", "libc", - "redox_syscall 0.1.57", "xattr", ] @@ -2922,7 +2944,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall 0.2.5", + "redox_syscall", "remove_dir_all", "winapi 0.3.9", ] @@ -2947,18 +2969,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ "proc-macro2", "quote", @@ -2967,11 +2989,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.0.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] @@ -2996,9 +3018,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +checksum = "1195b046942c221454c2539395f85413b33383a067449d78aab2b7b052a142f7" dependencies = [ "const_fn", "libc", @@ -3034,9 +3056,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" dependencies = [ "tinyvec_macros", ] @@ -3049,9 +3071,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" dependencies = [ "bytes 0.5.6", "fnv", @@ -3110,19 +3132,19 @@ dependencies = [ [[package]] name = "tower-service" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.22" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.4", "tracing-core", ] @@ -3137,11 +3159,11 @@ dependencies = [ [[package]] name = "tracing-futures" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 0.4.27", + "pin-project 1.0.5", "tracing", ] @@ -3232,9 +3254,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" dependencies = [ "tinyvec", ] @@ -3265,9 +3287,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" dependencies = [ "form_urlencoded", "idna", @@ -3300,12 +3322,13 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vergen" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" dependencies = [ "bitflags", "chrono", + "rustc_version 0.3.3", ] [[package]] @@ -3338,9 +3361,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" dependencies = [ "cfg-if 1.0.0", "serde", @@ -3350,9 +3373,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" dependencies = [ "bumpalo", "lazy_static", @@ -3365,9 +3388,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.19" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +checksum = "8e67a5806118af01f0d9045915676b22aaebecf4178ae7021bc171dab0b897ab" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3377,9 +3400,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3387,9 +3410,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" dependencies = [ "proc-macro2", "quote", @@ -3400,15 +3423,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" +checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" [[package]] name = "web-sys" -version = "0.3.46" +version = "0.3.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 875165588..a1dca038e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,79 +1,8 @@ -[package] -authors = ["Quentin de Quelen ", "Clément Renault ", "Marin Postma "] -description = "MeiliSearch HTTP server" -edition = "2018" -license = "MIT" -name = "meilisearch-http" -version = "0.17.0" -[[bin]] -name = "meilisearch" -path = "src/main.rs" +[workspace] +members = [ + "meilisearch-http", + "meilisearch-error", +] -[build-dependencies] -vergen = "3.1.0" - -[dependencies] -actix-cors = "0.5.3" -actix-http = "2" -actix-rt = "1" -actix-service = "1.0.6" -actix-web = { version = "3.3.2", features = ["rustls"] } -anyhow = "1.0.38" -async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } -byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } -bytes = "0.6.0" -chrono = { version = "0.4.19", features = ["serde"] } -crossbeam-channel = "0.5.0" -env_logger = "0.8.2" -flate2 = "1.0.19" -fst = "0.4.5" -futures = "0.3.7" -futures-util = "0.3.8" -grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = "0.10.6" -http = "0.2.1" -indexmap = { version = "1.3.2", features = ["serde-1"] } -log = "0.4.8" -main_error = "0.1.0" -meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } -memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "8dcb3e0" } -mime = "0.3.16" -once_cell = "1.5.2" -rand = "0.7.3" -rayon = "1.5.0" -regex = "1.4.2" -rustls = "0.18" -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0.59", features = ["preserve_order"] } -sha2 = "0.9.1" -siphasher = "0.3.2" -slice-group-by = "0.2.6" -structopt = "0.3.20" -tar = "0.4.29" -tempfile = "3.1.0" -tokio = { version = "0.2", features = ["full"] } -dashmap = "4.0.2" -uuid = "0.8.2" -itertools = "0.10.0" -either = "1.6.1" - -[dependencies.sentry] -default-features = false -features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] -optional = true -version = "0.18.1" - - -[dev-dependencies] -serde_url_params = "0.2.0" -tempdir = "0.3.7" -assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } -tokio = { version = "0.2", features = ["macros", "time"] } - -[features] -default = ["sentry"] - -[target.'cfg(unix)'.dependencies] -jemallocator = "0.3.2" +[profile.release] +debug = true diff --git a/meilisearch-error/Cargo.toml b/meilisearch-error/Cargo.toml new file mode 100644 index 000000000..d5a474ea5 --- /dev/null +++ b/meilisearch-error/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "meilisearch-error" +version = "0.19.0" +authors = ["marin "] +edition = "2018" + +[dependencies] +actix-http = "2.2.0" diff --git a/meilisearch-error/src/lib.rs b/meilisearch-error/src/lib.rs new file mode 100644 index 000000000..d0e00e9be --- /dev/null +++ b/meilisearch-error/src/lib.rs @@ -0,0 +1,185 @@ +use std::fmt; + +use actix_http::http::StatusCode; + +pub trait ErrorCode: std::error::Error { + fn error_code(&self) -> Code; + + /// returns the HTTP status code ascociated with the error + fn http_status(&self) -> StatusCode { + self.error_code().http() + } + + /// returns the doc url ascociated with the error + fn error_url(&self) -> String { + self.error_code().url() + } + + /// returns error name, used as error code + fn error_name(&self) -> String { + self.error_code().name() + } + + /// return the error type + fn error_type(&self) -> String { + self.error_code().type_() + } +} + +#[allow(clippy::enum_variant_names)] +enum ErrorType { + InternalError, + InvalidRequestError, + AuthenticationError, +} + +impl fmt::Display for ErrorType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use ErrorType::*; + + match self { + InternalError => write!(f, "internal_error"), + InvalidRequestError => write!(f, "invalid_request_error"), + AuthenticationError => write!(f, "authentication_error"), + } + } +} + +pub enum Code { + // index related error + CreateIndex, + IndexAlreadyExists, + IndexNotFound, + InvalidIndexUid, + OpenIndex, + + // invalid state error + InvalidState, + MissingPrimaryKey, + PrimaryKeyAlreadyPresent, + + MaxFieldsLimitExceeded, + MissingDocumentId, + + Facet, + Filter, + + BadParameter, + BadRequest, + DocumentNotFound, + Internal, + InvalidToken, + MissingAuthorizationHeader, + NotFound, + PayloadTooLarge, + RetrieveDocument, + SearchDocuments, + UnsupportedMediaType, + + DumpAlreadyInProgress, + DumpProcessFailed, +} + +impl Code { + + /// ascociate a `Code` variant to the actual ErrCode + fn err_code(&self) -> ErrCode { + use Code::*; + + match self { + // index related errors + // create index is thrown on internal error while creating an index. + CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST), + IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST), + // thrown when requesting an unexisting index + IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND), + InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST), + OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR), + + // invalid state error + InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR), + // thrown when no primary key has been set + MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST), + // error thrown when trying to set an already existing primary key + PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST), + + // invalid document + MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST), + MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST), + + // error related to facets + Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST), + // error related to filters + Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST), + + BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST), + BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST), + DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND), + Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR), + InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN), + MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED), + NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND), + PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE), + RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST), + SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST), + UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE), + + // error related to dump + DumpAlreadyInProgress => ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT), + DumpProcessFailed => ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR), + } + } + + /// return the HTTP status code ascociated with the `Code` + fn http(&self) -> StatusCode { + self.err_code().status_code + } + + /// return error name, used as error code + fn name(&self) -> String { + self.err_code().error_name.to_string() + } + + /// return the error type + fn type_(&self) -> String { + self.err_code().error_type.to_string() + } + + /// return the doc url ascociated with the error + fn url(&self) -> String { + format!("https://docs.meilisearch.com/errors#{}", self.name()) + } +} + +/// Internal structure providing a convenient way to create error codes +struct ErrCode { + status_code: StatusCode, + error_type: ErrorType, + error_name: &'static str, +} + +impl ErrCode { + fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::AuthenticationError, + } + } + + fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::InternalError, + } + } + + fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::InvalidRequestError, + } + } +} diff --git a/meilisearch-http/Cargo.lock b/meilisearch-http/Cargo.lock new file mode 100644 index 000000000..ec936a450 --- /dev/null +++ b/meilisearch-http/Cargo.lock @@ -0,0 +1,3581 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "actix-codec" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" +dependencies = [ + "bitflags", + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project 0.4.27", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-connect" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "derive_more", + "either", + "futures-util", + "http", + "log", + "rustls", + "tokio-rustls", + "trust-dns-proto", + "trust-dns-resolver", + "webpki", +] + +[[package]] +name = "actix-cors" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3a3d5493dbc9b8769fe88c030d057ef8d2edc5728e5e26267780e8fc5db0be" +dependencies = [ + "actix-web", + "derive_more", + "futures-util", + "log", + "once_cell", + "tinyvec", +] + +[[package]] +name = "actix-http" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" +dependencies = [ + "actix-codec", + "actix-connect", + "actix-rt", + "actix-service", + "actix-threadpool", + "actix-tls", + "actix-utils", + "base64 0.13.0", + "bitflags", + "brotli2", + "bytes 0.5.6", + "cookie", + "copyless", + "derive_more", + "either", + "encoding_rs", + "flate2", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "h2", + "http", + "httparse", + "indexmap", + "itoa", + "language-tags", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project 1.0.2", + "rand 0.7.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1 0.9.2", + "slab", + "time 0.2.23", +] + +[[package]] +name = "actix-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd1f7dbda1645bf7da33554db60891755f6c01c1b2169e2f4c492098d30c235" +dependencies = [ + "bytestring", + "http", + "log", + "regex", + "serde", +] + +[[package]] +name = "actix-rt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" +dependencies = [ + "actix-macros", + "actix-threadpool", + "copyless", + "futures-channel", + "futures-util", + "smallvec", + "tokio", +] + +[[package]] +name = "actix-server" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "futures-channel", + "futures-util", + "log", + "mio", + "mio-uds", + "num_cpus", + "slab", + "socket2", +] + +[[package]] +name = "actix-service" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" +dependencies = [ + "futures-util", + "pin-project 0.4.27", +] + +[[package]] +name = "actix-testing" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" +dependencies = [ + "actix-macros", + "actix-rt", + "actix-server", + "actix-service", + "log", + "socket2", +] + +[[package]] +name = "actix-threadpool" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" +dependencies = [ + "derive_more", + "futures-channel", + "lazy_static", + "log", + "num_cpus", + "parking_lot", + "threadpool", +] + +[[package]] +name = "actix-tls" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24789b7d7361cf5503a504ebe1c10806896f61e96eca9a7350e23001aca715fb" +dependencies = [ + "actix-codec", + "actix-service", + "actix-utils", + "futures-util", + "rustls", + "tokio-rustls", + "webpki", + "webpki-roots", +] + +[[package]] +name = "actix-utils" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "bitflags", + "bytes 0.5.6", + "either", + "futures-channel", + "futures-sink", + "futures-util", + "log", + "pin-project 0.4.27", + "slab", +] + +[[package]] +name = "actix-web" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-testing", + "actix-threadpool", + "actix-tls", + "actix-utils", + "actix-web-codegen", + "awc", + "bytes 0.5.6", + "derive_more", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "log", + "mime", + "pin-project 1.0.2", + "regex", + "rustls", + "serde", + "serde_json", + "serde_urlencoded", + "socket2", + "time 0.2.23", + "tinyvec", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad26f77093333e0e7c6ffe54ebe3582d908a104e448723eec6d43d08b07143fb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "addr2line" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "anyhow" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" + +[[package]] +name = "assert-json-diff" +version = "1.0.1" +source = "git+https://github.com/qdequele/assert-json-diff#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-compression" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite 0.1.11", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "awc" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" +dependencies = [ + "actix-codec", + "actix-http", + "actix-rt", + "actix-service", + "base64 0.13.0", + "bytes 0.5.6", + "cfg-if 1.0.0", + "derive_more", + "futures-core", + "log", + "mime", + "percent-encoding", + "rand 0.7.3", + "rustls", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "backtrace" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +dependencies = [ + "addr2line", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bincode" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" +dependencies = [ + "byteorder", + "serde", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "bstr" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byte-unit" +version = "4.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c8758c32833faaae35b24a73d332e62d0528e89076ae841c63940e37008b153" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "bytemuck" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" + +[[package]] +name = "bytestring" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7c05fa5172da78a62d9949d662d2ac89d4cc7355d7b49adee5163f1fb3f363" +dependencies = [ + "bytes 0.5.6", +] + +[[package]] +name = "cc" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cedarwood" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963e82c7b94163808ca3a452608d260b64ba5bc7b5653b4af1af59887899f48d" +dependencies = [ + "smallvec", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "character_converter" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e48477ece09d6a21c033cb604968524a37782532727055d6f6faafac1781e5c" +dependencies = [ + "bincode", +] + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "const_fn" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" + +[[package]] +name = "cookie" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +dependencies = [ + "percent-encoding", + "time 0.2.23", + "version_check", +] + +[[package]] +name = "copyless" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" + +[[package]] +name = "cow-utils" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.1", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils 0.8.1", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils 0.8.1", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "csv" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + +[[package]] +name = "debugid" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba" +dependencies = [ + "serde", + "uuid", +] + +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "deunicode" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +dependencies = [ + "atty", + "humantime 2.0.1", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "filetime" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.1.57", + "winapi 0.3.9", +] + +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + +[[package]] +name = "fst" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" + +[[package]] +name = "futures-executor" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" + +[[package]] +name = "futures-macro" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" + +[[package]] +name = "futures-task" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project 1.0.2", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "grenad" +version = "0.1.0" +source = "git+https://github.com/Kerollmops/grenad.git?rev=3adcb26#3adcb267dcbc590c7da10eb5f887a254865b3dbe" +dependencies = [ + "byteorder", + "flate2", + "log", + "nix", + "snap", + "tempfile", + "zstd", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "hashbrown" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" +dependencies = [ + "ahash", + "autocfg", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heed" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" +dependencies = [ + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-rkv-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", + "zerocopy", +] + +[[package]] +name = "heed-traits" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" + +[[package]] +name = "heed-types" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" +dependencies = [ + "bincode", + "heed-traits", + "serde", + "serde_json", + "zerocopy", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + +[[package]] +name = "http" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84129d298a6d57d246960ff8eb831ca4af3f96d29e2e28848dae275408658e26" +dependencies = [ + "bytes 0.5.6", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "human_format" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "humantime" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" + +[[package]] +name = "hyper" +version = "0.13.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.2", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" +dependencies = [ + "bytes 0.5.6", + "futures-util", + "hyper", + "log", + "rustls", + "tokio", + "tokio-rustls", + "webpki", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "im" +version = "14.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696059c87b83c5a258817ecd67c3af915e3ed141891fc35a1e79908801cf0ce7" +dependencies = [ + "bitmaps", + "rand_core 0.5.1", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "indexmap" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +dependencies = [ + "autocfg", + "hashbrown 0.9.1", + "serde", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "jemalloc-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", +] + +[[package]] +name = "jieba-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34fbdeee8786790f4a99fa30ff5c5f88aa5183f7583693e3788d17fc8a48f33a" +dependencies = [ + "cedarwood", + "fxhash", + "hashbrown 0.9.1", + "lazy_static", + "phf", + "phf_codegen", + "regex", +] + +[[package]] +name = "jobserver" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "levenshtein_automata" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a" +dependencies = [ + "fst", +] + +[[package]] +name = "libc" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "lmdb-rkv-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b27470ac25167b3afdfb6af8fcd3bc1be67de50ffbdaf4073378cfded6ae24a5" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "main_error" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb63bb1e282e0b6aba0addb1f0e87cb5181ea68142b2dfd21ba108f8e8088a64" + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "meilisearch-error" +version = "0.19.0" +dependencies = [ + "actix-http", +] + +[[package]] +name = "meilisearch-http" +version = "0.17.0" +dependencies = [ + "actix-cors", + "actix-http", + "actix-rt", + "actix-service", + "actix-web", + "anyhow", + "assert-json-diff", + "async-compression", + "byte-unit", + "bytes 0.6.0", + "chrono", + "crossbeam-channel", + "dashmap", + "either", + "env_logger 0.8.2", + "flate2", + "fst", + "futures", + "futures-util", + "grenad", + "heed", + "http", + "indexmap", + "itertools 0.10.0", + "jemallocator", + "log", + "main_error", + "meilisearch-error", + "meilisearch-tokenizer", + "memmap", + "milli", + "mime", + "once_cell", + "rand 0.7.3", + "rayon", + "regex", + "rustls", + "sentry", + "serde", + "serde_json", + "serde_url_params", + "sha2", + "siphasher", + "slice-group-by", + "structopt", + "tar", + "tempdir", + "tempfile", + "tokio", + "uuid", + "vergen", +] + +[[package]] +name = "meilisearch-tokenizer" +version = "0.1.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#147b6154b1b34cb8f5da2df6a416b7da191bc850" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + +[[package]] +name = "milli" +version = "0.1.0" +source = "git+https://github.com/meilisearch/milli.git?rev=8dcb3e0#8dcb3e0c41965c96ae718ae85c45004cf94c6e94" +dependencies = [ + "anyhow", + "bstr", + "byteorder", + "crossbeam-channel", + "csv", + "either", + "flate2", + "fst", + "fxhash", + "grenad", + "heed", + "human_format", + "itertools 0.10.0", + "levenshtein_automata", + "linked-hash-map", + "log", + "meilisearch-tokenizer", + "memmap", + "num-traits", + "obkv", + "once_cell", + "ordered-float", + "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", + "pest_derive", + "rayon", + "regex", + "roaring", + "serde", + "serde_json", + "smallstr", + "smallvec", + "tempfile", + "uuid", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio", + "miow 0.3.6", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nix" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" + +[[package]] +name = "obkv" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" + +[[package]] +name = "once_cell" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "ordered-float" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" +dependencies = [ + "num-traits", +] + +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest" +version = "2.1.3" +source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sha-1 0.8.2", +] + +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared", + "rand 0.7.3", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +dependencies = [ + "pin-project-internal 1.0.2", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-project-lite" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.15", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", + "rand_pcg", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.15", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils 0.8.1", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" +dependencies = [ + "base64 0.13.0", + "bytes 0.5.6", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite 0.2.0", + "rustls", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-rustls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "retain_mut" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" + +[[package]] +name = "ring" +version = "0.16.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "roaring" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6744a4a918e91359ad1d356a91e2e943a86d9fb9ae77f715d617032ea2af88f" +dependencies = [ + "bytemuck", + "byteorder", + "retain_mut", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64 0.12.3", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "sentry" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b01b723fc1b0a0f9394ca1a8451daec6e20206d47f96c3dceea7fd11ec9eec0" +dependencies = [ + "backtrace", + "env_logger 0.7.1", + "failure", + "hostname", + "httpdate", + "im", + "lazy_static", + "libc", + "log", + "rand 0.7.3", + "regex", + "reqwest", + "rustc_version", + "sentry-types", + "uname", + "url", +] + +[[package]] +name = "sentry-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ec406c11c060c8a7d5d67fc6f4beb2888338dcb12b9af409451995f124749d" +dependencies = [ + "chrono", + "debugid", + "failure", + "serde", + "serde_json", + "url", + "uuid", +] + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_url_params" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +dependencies = [ + "serde", + "url", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha-1" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" + +[[package]] +name = "sized-chunks" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59044ea371ad781ff976f7b06480b9f0180e834eda94114f2afb4afc12b7718" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "slice-group-by" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7474f0b646d228360ab62ed974744617bc869d959eac8403bfa3665931a7fb" + +[[package]] +name = "smallstr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e922794d168678729ffc7e07182721a14219c65814e66e91b839a272fe5ae4f" +dependencies = [ + "serde", + "smallvec", +] + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "snap" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98d3306e84bf86710d6cd8b4c9c3b721d5454cc91a603180f8f8cd06cfd317b4" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "standback" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" +dependencies = [ + "version_check", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synchronoise" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tar" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +dependencies = [ + "filetime", + "libc", + "redox_syscall 0.1.57", + "xattr", +] + +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall 0.2.5", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-named-pipes", + "mio-uds", + "num_cpus", + "pin-project-lite 0.1.11", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" +dependencies = [ + "futures-core", + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.1.11", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite 0.2.0", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "trust-dns-proto" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" +dependencies = [ + "async-trait", + "backtrace", + "enum-as-inner", + "futures", + "idna", + "lazy_static", + "log", + "rand 0.7.3", + "smallvec", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" +dependencies = [ + "backtrace", + "cfg-if 0.1.10", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "uname" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" +dependencies = [ + "libc", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-width" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.2", + "serde", +] + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "vergen" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +dependencies = [ + "bitflags", + "chrono", +] + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "web-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" +dependencies = [ + "webpki", +] + +[[package]] +name = "whatlang" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075" +dependencies = [ + "hashbrown 0.7.2", +] + +[[package]] +name = "widestring" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + +[[package]] +name = "zerocopy" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + +[[package]] +name = "zstd" +version = "0.5.4+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "2.0.6+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.4.18+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" +dependencies = [ + "cc", + "glob", + "itertools 0.9.0", + "libc", +] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml new file mode 100644 index 000000000..35ffb7771 --- /dev/null +++ b/meilisearch-http/Cargo.toml @@ -0,0 +1,79 @@ +[package] +authors = ["Quentin de Quelen ", "Clément Renault "] +description = "MeiliSearch HTTP server" +edition = "2018" +license = "MIT" +name = "meilisearch-http" +version = "0.17.0" +[[bin]] +name = "meilisearch" +path = "src/main.rs" + +[build-dependencies] +vergen = "3.1.0" + +[dependencies] +actix-cors = "0.5.3" +actix-http = "2" +actix-rt = "1" +actix-service = "1.0.6" +actix-web = { version = "3.3.2", features = ["rustls"] } +anyhow = "1.0.38" +async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } +byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } +bytes = "0.6.0" +chrono = { version = "0.4.19", features = ["serde"] } +crossbeam-channel = "0.5.0" +env_logger = "0.8.2" +flate2 = "1.0.19" +fst = "0.4.5" +futures = "0.3.7" +futures-util = "0.3.8" +grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } +heed = "0.10.6" +http = "0.2.1" +indexmap = { version = "1.3.2", features = ["serde-1"] } +log = "0.4.8" +main_error = "0.1.0" +meilisearch-error = { path = "../meilisearch-error" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } +memmap = "0.7.0" +milli = { git = "https://github.com/meilisearch/milli.git", rev = "8dcb3e0" } +mime = "0.3.16" +once_cell = "1.5.2" +rand = "0.7.3" +rayon = "1.5.0" +regex = "1.4.2" +rustls = "0.18" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0.59", features = ["preserve_order"] } +sha2 = "0.9.1" +siphasher = "0.3.2" +slice-group-by = "0.2.6" +structopt = "0.3.20" +tar = "0.4.29" +tempfile = "3.1.0" +tokio = { version = "0.2", features = ["full"] } +dashmap = "4.0.2" +uuid = "0.8.2" +itertools = "0.10.0" +either = "1.6.1" + +[dependencies.sentry] +default-features = false +features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] +optional = true +version = "0.18.1" + + +[dev-dependencies] +serde_url_params = "0.2.0" +tempdir = "0.3.7" +assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } +tokio = { version = "0.2", features = ["macros", "time"] } + +[features] +default = ["sentry"] + +[target.'cfg(unix)'.dependencies] +jemallocator = "0.3.2" diff --git a/meilisearch-http/_tests/dashboard.rs b/meilisearch-http/_tests/dashboard.rs new file mode 100644 index 000000000..2dbaf8f7d --- /dev/null +++ b/meilisearch-http/_tests/dashboard.rs @@ -0,0 +1,12 @@ +mod common; + +#[actix_rt::test] +async fn dashboard() { + let mut server = common::Server::with_uid("movies"); + + let (_response, status_code) = server.get_request("/").await; + assert_eq!(status_code, 200); + + let (_response, status_code) = server.get_request("/bulma.min.css").await; + assert_eq!(status_code, 200); +} diff --git a/meilisearch-http/_tests/documents_add.rs b/meilisearch-http/_tests/documents_add.rs new file mode 100644 index 000000000..382a1ed43 --- /dev/null +++ b/meilisearch-http/_tests/documents_add.rs @@ -0,0 +1,222 @@ +use serde_json::json; + +mod common; + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/519 +#[actix_rt::test] +async fn check_add_documents_with_primary_key_param() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add documents + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/movies/documents?primaryKey=title"; + let (response, status_code) = server.post_request(&url, body).await; + eprintln!("{:#?}", response); + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/568 +#[actix_rt::test] +async fn check_add_documents_with_nested_boolean() { + let mut server = common::Server::with_uid("tasks"); + + // 1 - Create the index with no primary_key + + let body = json!({ "uid": "tasks" }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add a document that contains a boolean in a nested object + + let body = json!([{ + "id": 12161, + "created_at": "2019-04-10T14:57:57.522Z", + "foo": { + "bar": { + "id": 121, + "crash": false + }, + "id": 45912 + } + }]); + + let url = "/indexes/tasks/documents"; + let (response, status_code) = server.post_request(&url, body).await; + eprintln!("{:#?}", response); + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/571 +#[actix_rt::test] +async fn check_add_documents_with_nested_null() { + let mut server = common::Server::with_uid("tasks"); + + // 1 - Create the index with no primary_key + + let body = json!({ "uid": "tasks" }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add a document that contains a null in a nested object + + let body = json!([{ + "id": 0, + "foo": { + "bar": null + } + }]); + + let url = "/indexes/tasks/documents"; + let (response, status_code) = server.post_request(&url, body).await; + eprintln!("{:#?}", response); + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/574 +#[actix_rt::test] +async fn check_add_documents_with_nested_sequence() { + let mut server = common::Server::with_uid("tasks"); + + // 1 - Create the index with no primary_key + + let body = json!({ "uid": "tasks" }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add a document that contains a seq in a nested object + + let body = json!([{ + "id": 0, + "foo": { + "bar": [123,456], + "fez": [{ + "id": 255, + "baz": "leesz", + "fuzz": { + "fax": [234] + }, + "sas": [] + }], + "foz": [{ + "id": 255, + "baz": "leesz", + "fuzz": { + "fax": [234] + }, + "sas": [] + }, + { + "id": 256, + "baz": "loss", + "fuzz": { + "fax": [235] + }, + "sas": [321, 321] + }] + } + }]); + + let url = "/indexes/tasks/documents"; + let (response, status_code) = server.post_request(&url, body.clone()).await; + eprintln!("{:#?}", response); + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); + + let url = "/indexes/tasks/search?q=leesz"; + let (response, status_code) = server.get_request(&url).await; + assert_eq!(status_code, 200); + assert_eq!(response["hits"], body); +} + +#[actix_rt::test] +// test sample from #807 +async fn add_document_with_long_field() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test" })).await; + let body = json!([{ + "documentId":"de1c2adbb897effdfe0deae32a01035e46f932ce", + "rank":1, + "relurl":"/configuration/app/web.html#locations", + "section":"Web", + "site":"docs", + "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", + "title":"Locations", + "url":"/configuration/app/web.html#locations" + }]); + server.add_or_replace_multiple_documents(body).await; + let (response, _status) = server + .search_post(json!({ "q": "request_buffering" })) + .await; + assert!(!response["hits"].as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn documents_with_same_id_are_overwritten() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test"})).await; + let documents = json!([ + { + "id": 1, + "content": "test1" + }, + { + "id": 1, + "content": "test2" + }, + ]); + server.add_or_replace_multiple_documents(documents).await; + let (response, _status) = server.get_all_documents().await; + assert_eq!(response.as_array().unwrap().len(), 1); + assert_eq!( + response.as_array().unwrap()[0].as_object().unwrap()["content"], + "test2" + ); +} diff --git a/meilisearch-http/_tests/documents_delete.rs b/meilisearch-http/_tests/documents_delete.rs new file mode 100644 index 000000000..4353a5355 --- /dev/null +++ b/meilisearch-http/_tests/documents_delete.rs @@ -0,0 +1,67 @@ +mod common; + +use serde_json::json; + +#[actix_rt::test] +async fn delete() { + let mut server = common::Server::test_server().await; + + let (_response, status_code) = server.get_document(50).await; + assert_eq!(status_code, 200); + + server.delete_document(50).await; + + let (_response, status_code) = server.get_document(50).await; + assert_eq!(status_code, 404); +} + +// Resolve the issue https://github.com/meilisearch/MeiliSearch/issues/493 +#[actix_rt::test] +async fn delete_batch() { + let mut server = common::Server::test_server().await; + + let doc_ids = vec!(50, 55, 60); + for doc_id in &doc_ids { + let (_response, status_code) = server.get_document(doc_id).await; + assert_eq!(status_code, 200); + } + + let body = serde_json::json!(&doc_ids); + server.delete_multiple_documents(body).await; + + for doc_id in &doc_ids { + let (_response, status_code) = server.get_document(doc_id).await; + assert_eq!(status_code, 404); + } +} + +#[actix_rt::test] +async fn text_clear_all_placeholder_search() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + }); + + server.create_index(body).await; + let settings = json!({ + "attributesForFaceting": ["genre"], + }); + + server.update_all_settings(settings).await; + + let documents = json!([ + { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance" }, + { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" }, + { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy" }, + { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, + { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, + { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams" } + ]); + + server.add_or_update_multiple_documents(documents).await; + server.clear_all_documents().await; + let (response, _) = server.search_post(json!({ "q": "", "facetsDistribution": ["genre"] })).await; + assert_eq!(response["nbHits"], 0); + let (response, _) = server.search_post(json!({ "q": "" })).await; + assert_eq!(response["nbHits"], 0); +} diff --git a/meilisearch-http/_tests/documents_get.rs b/meilisearch-http/_tests/documents_get.rs new file mode 100644 index 000000000..35e04f494 --- /dev/null +++ b/meilisearch-http/_tests/documents_get.rs @@ -0,0 +1,23 @@ +use serde_json::json; +use actix_web::http::StatusCode; + +mod common; + +#[actix_rt::test] +async fn get_documents_from_unexisting_index_is_error() { + let mut server = common::Server::with_uid("test"); + let (response, status) = server.get_all_documents().await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(response["errorCode"], "index_not_found"); + assert_eq!(response["errorType"], "invalid_request_error"); + assert_eq!(response["errorLink"], "https://docs.meilisearch.com/errors#index_not_found"); +} + +#[actix_rt::test] +async fn get_empty_documents_list() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test" })).await; + let (response, status) = server.get_all_documents().await; + assert_eq!(status, StatusCode::OK); + assert!(response.as_array().unwrap().is_empty()); +} diff --git a/meilisearch-http/_tests/dump.rs b/meilisearch-http/_tests/dump.rs new file mode 100644 index 000000000..701b754aa --- /dev/null +++ b/meilisearch-http/_tests/dump.rs @@ -0,0 +1,395 @@ +use assert_json_diff::{assert_json_eq, assert_json_include}; +use meilisearch_http::helpers::compression; +use serde_json::{json, Value}; +use std::fs::File; +use std::path::Path; +use std::thread; +use std::time::Duration; +use tempfile::TempDir; + +#[macro_use] mod common; + +async fn trigger_and_wait_dump(server: &mut common::Server) -> String { + let (value, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + for _ in 0..20 as u8 { + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + assert_eq!(status_code, 200); + assert_ne!(value["status"].as_str(), Some("dump_process_failed")); + + if value["status"].as_str() == Some("done") { return dump_uid } + thread::sleep(Duration::from_millis(100)); + } + + unreachable!("dump creation runned out of time") +} + +fn current_db_version() -> (String, String, String) { + let current_version_major = env!("CARGO_PKG_VERSION_MAJOR").to_string(); + let current_version_minor = env!("CARGO_PKG_VERSION_MINOR").to_string(); + let current_version_patch = env!("CARGO_PKG_VERSION_PATCH").to_string(); + + (current_version_major, current_version_minor, current_version_patch) +} + +fn current_dump_version() -> String { + "V1".into() +} + +fn read_all_jsonline(r: R) -> Value { + let deserializer = serde_json::Deserializer::from_reader(r); + let iterator = deserializer.into_iter::(); + + json!(iterator.map(|v| v.unwrap()).collect::>()) +} + +#[actix_rt::test] +#[ignore] +async fn trigger_dump_should_return_ok() { + let server = common::Server::test_server().await; + + let (_, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); +} + +#[actix_rt::test] +#[ignore] +async fn trigger_dump_twice_should_return_conflict() { + let server = common::Server::test_server().await; + + let expected = json!({ + "message": "Another dump is already in progress", + "errorCode": "dump_already_in_progress", + "errorType": "invalid_request_error", + "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" + }); + + let (_, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); + + let (value, status_code) = server.trigger_dump().await; + + + assert_json_eq!(expected, value, ordered: false); + assert_eq!(status_code, 409); +} + +#[actix_rt::test] +#[ignore] +async fn trigger_dump_concurently_should_return_conflict() { + let server = common::Server::test_server().await; + + let expected = json!({ + "message": "Another dump is already in progress", + "errorCode": "dump_already_in_progress", + "errorType": "invalid_request_error", + "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" + }); + + let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump()); + + assert_json_eq!(expected, value_2, ordered: false); + assert_eq!(status_code_2, 409); +} + +#[actix_rt::test] +#[ignore] +async fn get_dump_status_early_should_return_in_progress() { + let mut server = common::Server::test_server().await; + + + + let (value, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + let expected = json!({ + "uid": dump_uid, + "status": "in_progress" + }); + + assert_eq!(status_code, 200); + + assert_json_eq!(expected, value, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn get_dump_status_should_return_done() { + let mut server = common::Server::test_server().await; + + + let (value, status_code) = server.trigger_dump().await; + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + let expected = json!({ + "uid": dump_uid.clone(), + "status": "done" + }); + + thread::sleep(Duration::from_secs(1)); // wait dump until process end + + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + assert_eq!(status_code, 200); + + assert_json_eq!(expected, value, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn get_dump_status_should_return_error_provoking_it() { + let mut server = common::Server::test_server().await; + + + let (value, status_code) = server.trigger_dump().await; + + // removing destination directory provoking `No such file or directory` error + std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap(); + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + let expected = json!({ + "uid": dump_uid.clone(), + "status": "failed", + "message": "Dump process failed: compressing dump; No such file or directory (os error 2)", + "errorCode": "dump_process_failed", + "errorType": "internal_error", + "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed" + }); + + thread::sleep(Duration::from_secs(1)); // wait dump until process end + + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + assert_eq!(status_code, 200); + + assert_json_eq!(expected, value, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn dump_metadata_should_be_valid() { + let mut server = common::Server::test_server().await; + + let body = json!({ + "uid": "test2", + "primaryKey": "test2_id", + }); + + server.create_index(body).await; + + let uid = trigger_and_wait_dump(&mut server).await; + + let dumps_dir = Path::new(&server.data().dumps_dir); + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path(); + + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); + + let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); + let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); + + // fields are randomly ordered + metadata.get_mut("indexes").unwrap() + .as_array_mut().unwrap() + .sort_by(|a, b| + a.get("uid").unwrap().as_str().cmp(&b.get("uid").unwrap().as_str()) + ); + + let (major, minor, patch) = current_db_version(); + + let expected = json!({ + "indexes": [{ + "uid": "test", + "primaryKey": "id", + }, { + "uid": "test2", + "primaryKey": "test2_id", + } + ], + "dbVersion": format!("{}.{}.{}", major, minor, patch), + "dumpVersion": current_dump_version() + }); + + assert_json_include!(expected: expected, actual: metadata); +} + +#[actix_rt::test] +#[ignore] +async fn dump_gzip_should_have_been_created() { + let mut server = common::Server::test_server().await; + + + let dump_uid = trigger_and_wait_dump(&mut server).await; + let dumps_dir = Path::new(&server.data().dumps_dir); + + let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid)); + assert!(File::open(compressed_path).is_ok()); +} + +#[actix_rt::test] +#[ignore] +async fn dump_index_settings_should_be_valid() { + let mut server = common::Server::test_server().await; + + let expected = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": "email", + "searchableAttributes": [ + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "displayedAttributes": [ + "id", + "isActive", + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "stopWords": [ + "in", + "ad" + ], + "synonyms": { + "wolverine": ["xmen", "logan"], + "logan": ["wolverine", "xmen"] + }, + "attributesForFaceting": [ + "gender", + "color", + "tags" + ] + }); + + server.update_all_settings(expected.clone()).await; + + let uid = trigger_and_wait_dump(&mut server).await; + + let dumps_dir = Path::new(&server.data().dumps_dir); + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path(); + + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); + + let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); + let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); + + assert_json_eq!(expected, settings, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn dump_index_documents_should_be_valid() { + let mut server = common::Server::test_server().await; + + let dataset = include_bytes!("assets/dumps/v1/test/documents.jsonl"); + let mut slice: &[u8] = dataset; + + let expected: Value = read_all_jsonline(&mut slice); + + let uid = trigger_and_wait_dump(&mut server).await; + + let dumps_dir = Path::new(&server.data().dumps_dir); + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path(); + + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); + + let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); + let documents = read_all_jsonline(file); + + assert_json_eq!(expected, documents, ordered: false); +} + +#[actix_rt::test] +#[ignore] +async fn dump_index_updates_should_be_valid() { + let mut server = common::Server::test_server().await; + + let dataset = include_bytes!("assets/dumps/v1/test/updates.jsonl"); + let mut slice: &[u8] = dataset; + + let expected: Value = read_all_jsonline(&mut slice); + + let uid = trigger_and_wait_dump(&mut server).await; + + let dumps_dir = Path::new(&server.data().dumps_dir); + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path(); + + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); + + let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); + let mut updates = read_all_jsonline(file); + + + // hotfix until #943 is fixed (https://github.com/meilisearch/MeiliSearch/issues/943) + updates.as_array_mut().unwrap() + .get_mut(0).unwrap() + .get_mut("type").unwrap() + .get_mut("settings").unwrap() + .get_mut("displayed_attributes").unwrap() + .get_mut("Update").unwrap() + .as_array_mut().unwrap().sort_by(|a, b| a.as_str().cmp(&b.as_str())); + + eprintln!("{}\n", updates.to_string()); + eprintln!("{}", expected.to_string()); + assert_json_include!(expected: expected, actual: updates); +} + +#[actix_rt::test] +#[ignore] +async fn get_unexisting_dump_status_should_return_not_found() { + let mut server = common::Server::test_server().await; + + let (_, status_code) = server.get_dump_status("4242").await; + + assert_eq!(status_code, 404); +} diff --git a/meilisearch-http/_tests/errors.rs b/meilisearch-http/_tests/errors.rs new file mode 100644 index 000000000..e11483356 --- /dev/null +++ b/meilisearch-http/_tests/errors.rs @@ -0,0 +1,200 @@ +mod common; + +use std::thread; +use std::time::Duration; + +use actix_http::http::StatusCode; +use serde_json::{json, Map, Value}; + +macro_rules! assert_error { + ($code:literal, $type:literal, $status:path, $req:expr) => { + let (response, status_code) = $req; + assert_eq!(status_code, $status); + assert_eq!(response["errorCode"].as_str().unwrap(), $code); + assert_eq!(response["errorType"].as_str().unwrap(), $type); + }; +} + +macro_rules! assert_error_async { + ($code:literal, $type:literal, $server:expr, $req:expr) => { + let (response, _) = $req; + let update_id = response["updateId"].as_u64().unwrap(); + for _ in 1..10 { + let (response, status_code) = $server.get_update_status(update_id).await; + assert_eq!(status_code, StatusCode::OK); + if response["status"] == "processed" || response["status"] == "failed" { + println!("response: {}", response); + assert_eq!(response["status"], "failed"); + assert_eq!(response["errorCode"], $code); + assert_eq!(response["errorType"], $type); + return + } + thread::sleep(Duration::from_secs(1)); + } + }; +} + +#[actix_rt::test] +async fn index_already_exists_error() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test" + }); + let (response, status_code) = server.create_index(body.clone()).await; + println!("{}", response); + assert_eq!(status_code, StatusCode::CREATED); + + let (response, status_code) = server.create_index(body.clone()).await; + println!("{}", response); + + assert_error!( + "index_already_exists", + "invalid_request_error", + StatusCode::BAD_REQUEST, + (response, status_code)); +} + +#[actix_rt::test] +async fn index_not_found_error() { + let mut server = common::Server::with_uid("test"); + assert_error!( + "index_not_found", + "invalid_request_error", + StatusCode::NOT_FOUND, + server.get_index().await); +} + +#[actix_rt::test] +async fn primary_key_already_present_error() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "test" + }); + server.create_index(body.clone()).await; + let body = json!({ + "primaryKey": "t" + }); + assert_error!( + "primary_key_already_present", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.update_index(body).await); +} + +#[actix_rt::test] +async fn max_field_limit_exceeded_error() { + let mut server = common::Server::test_server().await; + let body = json!({ + "uid": "test", + }); + server.create_index(body).await; + let mut doc = Map::with_capacity(70_000); + doc.insert("id".into(), Value::String("foo".into())); + for i in 0..69_999 { + doc.insert(format!("field{}", i), Value::String("foo".into())); + } + let docs = json!([doc]); + assert_error_async!( + "max_fields_limit_exceeded", + "invalid_request_error", + server, + server.add_or_replace_multiple_documents_sync(docs).await); +} + +#[actix_rt::test] +async fn missing_document_id() { + let mut server = common::Server::test_server().await; + let body = json!({ + "uid": "test", + "primaryKey": "test" + }); + server.create_index(body).await; + let docs = json!([ + { + "foo": "bar", + } + ]); + assert_error_async!( + "missing_document_id", + "invalid_request_error", + server, + server.add_or_replace_multiple_documents_sync(docs).await); +} + +#[actix_rt::test] +async fn facet_error() { + let mut server = common::Server::test_server().await; + let search = json!({ + "q": "foo", + "facetFilters": ["test:hello"] + }); + assert_error!( + "invalid_facet", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.search_post(search).await); +} + +#[actix_rt::test] +async fn filters_error() { + let mut server = common::Server::test_server().await; + let search = json!({ + "q": "foo", + "filters": "fo:12" + }); + assert_error!( + "invalid_filter", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.search_post(search).await); +} + +#[actix_rt::test] +async fn bad_request_error() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "foo": "bar", + }); + assert_error!( + "bad_request", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.search_post(body).await); +} + +#[actix_rt::test] +async fn document_not_found_error() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({"uid": "test"})).await; + assert_error!( + "document_not_found", + "invalid_request_error", + StatusCode::NOT_FOUND, + server.get_document(100).await); +} + +#[actix_rt::test] +async fn payload_too_large_error() { + let mut server = common::Server::with_uid("test"); + let bigvec = vec![0u64; 10_000_000]; // 80mb + assert_error!( + "payload_too_large", + "invalid_request_error", + StatusCode::PAYLOAD_TOO_LARGE, + server.create_index(json!(bigvec)).await); +} + +#[actix_rt::test] +async fn missing_primary_key_error() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({"uid": "test"})).await; + let document = json!([{ + "content": "test" + }]); + assert_error!( + "missing_primary_key", + "invalid_request_error", + StatusCode::BAD_REQUEST, + server.add_or_replace_multiple_documents_sync(document).await); +} diff --git a/meilisearch-http/_tests/health.rs b/meilisearch-http/_tests/health.rs new file mode 100644 index 000000000..f72127431 --- /dev/null +++ b/meilisearch-http/_tests/health.rs @@ -0,0 +1,11 @@ +mod common; + +#[actix_rt::test] +async fn test_healthyness() { + let mut server = common::Server::with_uid("movies"); + + // Check that the server is healthy + + let (_response, status_code) = server.get_health().await; + assert_eq!(status_code, 204); +} diff --git a/meilisearch-http/_tests/index.rs b/meilisearch-http/_tests/index.rs new file mode 100644 index 000000000..271507e03 --- /dev/null +++ b/meilisearch-http/_tests/index.rs @@ -0,0 +1,809 @@ +use actix_web::http::StatusCode; +use assert_json_diff::assert_json_eq; +use serde_json::{json, Value}; + +mod common; + +#[actix_rt::test] +async fn create_index_with_name() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "name": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 8); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid.len(), r1_uid.len()); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); +} + +#[actix_rt::test] +async fn create_index_with_uid() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "uid": "movies", + }); + + let (res1_value, status_code) = server.create_index(body.clone()).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid, "movies"); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 1.5 verify that error is thrown when trying to create the same index + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + assert_eq!( + response["errorCode"].as_str().unwrap(), + "index_already_exists" + ); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid, r1_uid); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); +} + +#[actix_rt::test] +async fn create_index_with_name_and_uid() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "name": "Films", + "uid": "fr_movies", + }); + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "Films"); + assert_eq!(r1_uid, "fr_movies"); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid, r1_uid); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); +} + +#[actix_rt::test] +async fn rename_index() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "name": "movies", + "uid": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 6); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Update an index name + + let body = json!({ + "name": "TV Shows", + }); + + let (res2_value, status_code) = server.update_index(body).await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_object().unwrap().len(), 5); + let r2_name = res2_value["name"].as_str().unwrap(); + let r2_uid = res2_value["uid"].as_str().unwrap(); + let r2_created_at = res2_value["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, "TV Shows"); + assert_eq!(r2_uid, r1_uid); + assert_eq!(r2_created_at, r1_created_at); + assert!(r2_updated_at.len() > 1); + + // 3 - Check the list of indexes + + let (res3_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res3_value.as_array().unwrap().len(), 1); + assert_eq!(res3_value[0].as_object().unwrap().len(), 5); + let r3_name = res3_value[0]["name"].as_str().unwrap(); + let r3_uid = res3_value[0]["uid"].as_str().unwrap(); + let r3_created_at = res3_value[0]["createdAt"].as_str().unwrap(); + let r3_updated_at = res3_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r3_name, r2_name); + assert_eq!(r3_uid.len(), r1_uid.len()); + assert_eq!(r3_created_at.len(), r1_created_at.len()); + assert_eq!(r3_updated_at.len(), r2_updated_at.len()); +} + +#[actix_rt::test] +async fn delete_index_and_recreate_it() { + let mut server = common::Server::with_uid("movies"); + + // 0 - delete unexisting index is error + + let (response, status_code) = server.delete_request("/indexes/test").await; + assert_eq!(status_code, 404); + assert_eq!(&response["errorCode"], "index_not_found"); + + // 1 - Create a new index + + let body = json!({ + "name": "movies", + "uid": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 6); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid.len(), r1_uid.len()); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); + + // 3- Delete an index + + let (_res2_value, status_code) = server.delete_index().await; + + assert_eq!(status_code, 204); + + // 4 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 0); + + // 5 - Create a new index + + let body = json!({ + "name": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 8); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 6 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_name = res2_value[0]["name"].as_str().unwrap(); + let r2_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_name, r1_name); + assert_eq!(r2_uid.len(), r1_uid.len()); + assert_eq!(r2_created_at.len(), r1_created_at.len()); + assert_eq!(r2_updated_at.len(), r1_updated_at.len()); +} + +#[actix_rt::test] +async fn check_multiples_indexes() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create a new index + + let body = json!({ + "name": "movies", + }); + + let (res1_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res1_value.as_object().unwrap().len(), 5); + let r1_name = res1_value["name"].as_str().unwrap(); + let r1_uid = res1_value["uid"].as_str().unwrap(); + let r1_created_at = res1_value["createdAt"].as_str().unwrap(); + let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); + assert_eq!(r1_name, "movies"); + assert_eq!(r1_uid.len(), 8); + assert!(r1_created_at.len() > 1); + assert!(r1_updated_at.len() > 1); + + // 2 - Check the list of indexes + + let (res2_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res2_value.as_array().unwrap().len(), 1); + assert_eq!(res2_value[0].as_object().unwrap().len(), 5); + let r2_0_name = res2_value[0]["name"].as_str().unwrap(); + let r2_0_uid = res2_value[0]["uid"].as_str().unwrap(); + let r2_0_created_at = res2_value[0]["createdAt"].as_str().unwrap(); + let r2_0_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(r2_0_name, r1_name); + assert_eq!(r2_0_uid.len(), r1_uid.len()); + assert_eq!(r2_0_created_at.len(), r1_created_at.len()); + assert_eq!(r2_0_updated_at.len(), r1_updated_at.len()); + + // 3 - Create a new index + + let body = json!({ + "name": "films", + }); + + let (res3_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 201); + assert_eq!(res3_value.as_object().unwrap().len(), 5); + let r3_name = res3_value["name"].as_str().unwrap(); + let r3_uid = res3_value["uid"].as_str().unwrap(); + let r3_created_at = res3_value["createdAt"].as_str().unwrap(); + let r3_updated_at = res3_value["updatedAt"].as_str().unwrap(); + assert_eq!(r3_name, "films"); + assert_eq!(r3_uid.len(), 8); + assert!(r3_created_at.len() > 1); + assert!(r3_updated_at.len() > 1); + + // 4 - Check the list of indexes + + let (res4_value, status_code) = server.list_indexes().await; + + assert_eq!(status_code, 200); + assert_eq!(res4_value.as_array().unwrap().len(), 2); + assert_eq!(res4_value[0].as_object().unwrap().len(), 5); + let r4_0_name = res4_value[0]["name"].as_str().unwrap(); + let r4_0_uid = res4_value[0]["uid"].as_str().unwrap(); + let r4_0_created_at = res4_value[0]["createdAt"].as_str().unwrap(); + let r4_0_updated_at = res4_value[0]["updatedAt"].as_str().unwrap(); + assert_eq!(res4_value[1].as_object().unwrap().len(), 5); + let r4_1_name = res4_value[1]["name"].as_str().unwrap(); + let r4_1_uid = res4_value[1]["uid"].as_str().unwrap(); + let r4_1_created_at = res4_value[1]["createdAt"].as_str().unwrap(); + let r4_1_updated_at = res4_value[1]["updatedAt"].as_str().unwrap(); + if r4_0_name == r1_name { + assert_eq!(r4_0_name, r1_name); + assert_eq!(r4_0_uid.len(), r1_uid.len()); + assert_eq!(r4_0_created_at.len(), r1_created_at.len()); + assert_eq!(r4_0_updated_at.len(), r1_updated_at.len()); + } else { + assert_eq!(r4_0_name, r3_name); + assert_eq!(r4_0_uid.len(), r3_uid.len()); + assert_eq!(r4_0_created_at.len(), r3_created_at.len()); + assert_eq!(r4_0_updated_at.len(), r3_updated_at.len()); + } + if r4_1_name == r1_name { + assert_eq!(r4_1_name, r1_name); + assert_eq!(r4_1_uid.len(), r1_uid.len()); + assert_eq!(r4_1_created_at.len(), r1_created_at.len()); + assert_eq!(r4_1_updated_at.len(), r1_updated_at.len()); + } else { + assert_eq!(r4_1_name, r3_name); + assert_eq!(r4_1_uid.len(), r3_uid.len()); + assert_eq!(r4_1_created_at.len(), r3_created_at.len()); + assert_eq!(r4_1_updated_at.len(), r3_updated_at.len()); + } +} + +#[actix_rt::test] +async fn create_index_failed() { + let mut server = common::Server::with_uid("movies"); + + // 2 - Push index creation with empty json body + + let body = json!({}); + + let (res_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = res_value["message"].as_str().unwrap(); + assert_eq!(res_value.as_object().unwrap().len(), 4); + assert_eq!(message, "Index creation must have an uid"); + + // 3 - Create a index with extra data + + let body = json!({ + "name": "movies", + "active": true + }); + + let (_res_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + + // 3 - Create a index with wrong data type + + let body = json!({ + "name": "movies", + "uid": 0 + }); + + let (_res_value, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); +} + +// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/492 +#[actix_rt::test] +async fn create_index_with_primary_key_and_index() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index + + let body = json!({ + "uid": "movies", + "primaryKey": "id", + }); + + let (_response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + + // 2 - Add content + + let body = json!([{ + "id": 123, + "text": "The mask" + }]); + + server.add_or_replace_multiple_documents(body.clone()).await; + + // 3 - Retreive document + + let (response, _status_code) = server.get_document(123).await; + + let expect = json!({ + "id": 123, + "text": "The mask" + }); + + assert_json_eq!(response, expect, ordered: false); +} + +// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/497 +// Test when the given index uid is not valid +// Should have a 400 status code +// Should have the right error message +#[actix_rt::test] +async fn create_index_with_invalid_uid() { + let mut server = common::Server::with_uid(""); + + // 1 - Create the index with invalid uid + + let body = json!({ + "uid": "the movies" + }); + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = response["message"].as_str().unwrap(); + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); + + // 2 - Create the index with invalid uid + + let body = json!({ + "uid": "%$#" + }); + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = response["message"].as_str().unwrap(); + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); + + // 3 - Create the index with invalid uid + + let body = json!({ + "uid": "the~movies" + }); + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = response["message"].as_str().unwrap(); + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); + + // 4 - Create the index with invalid uid + + let body = json!({ + "uid": "🎉" + }); + + let (response, status_code) = server.create_index(body).await; + + assert_eq!(status_code, 400); + let message = response["message"].as_str().unwrap(); + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); +} + +// Test that it's possible to add primary_key if it's not already set on index creation +#[actix_rt::test] +async fn create_index_and_add_indentifier_after() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Update the index and add an primary_key. + + let body = json!({ + "primaryKey": "id", + }); + + let (response, status_code) = server.update_index(body).await; + assert_eq!(status_code, 200); + eprintln!("response: {:#?}", response); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); + + // 3 - Get index to verify if the primary_key is good + + let (response, status_code) = server.get_index().await; + assert_eq!(status_code, 200); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); +} + +// Test that it's impossible to change the primary_key +#[actix_rt::test] +async fn create_index_and_update_indentifier_after() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + "primaryKey": "id", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); + + // 2 - Update the index and add an primary_key. + + let body = json!({ + "primaryKey": "skuid", + }); + + let (_response, status_code) = server.update_index(body).await; + assert_eq!(status_code, 400); + + // 3 - Get index to verify if the primary_key still the first one + + let (response, status_code) = server.get_index().await; + assert_eq!(status_code, 200); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); +} + +// Test that schema inference work well +#[actix_rt::test] +async fn create_index_without_primary_key_and_add_document() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Add a document + + let body = json!([{ + "id": 123, + "title": "I'm a legend", + }]); + + server.add_or_update_multiple_documents(body).await; + + // 3 - Get index to verify if the primary_key is good + + let (response, status_code) = server.get_index().await; + assert_eq!(status_code, 200); + assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); +} + +// Test search with no primary_key +#[actix_rt::test] +async fn create_index_without_primary_key_and_search() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2 - Search + + let query = "q=captain&limit=3"; + + let (response, status_code) = server.search_get(&query).await; + assert_eq!(status_code, 200); + assert_eq!(response["hits"].as_array().unwrap().len(), 0); +} + +// Test the error message when we push an document update and impossibility to find primary key +// Test issue https://github.com/meilisearch/MeiliSearch/issues/517 +#[actix_rt::test] +async fn check_add_documents_without_primary_key() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Create the index with no primary_key + + let body = json!({ + "uid": "movies", + }); + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2- Add document + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let (response, status_code) = server.add_or_replace_multiple_documents_sync(body).await; + + assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(response["errorCode"], "missing_primary_key"); + assert_eq!(status_code, 400); +} + +#[actix_rt::test] +async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { + let mut server = common::Server::with_uid("movies"); + + let body = json!({ + "uid": "movies", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + let dataset = include_bytes!("./assets/test_set.json"); + + let body: Value = serde_json::from_slice(dataset).unwrap(); + + // 2. Index the documents from movies.json, present inside of assets directory + server.add_or_replace_multiple_documents(body).await; + + // 3. Fetch the status of the indexing done above. + let (response, status_code) = server.get_all_updates_status().await; + + // 4. Verify the fetch is successful and indexing status is 'processed' + assert_eq!(status_code, 200); + assert_eq!(response[0]["status"], "processed"); +} + +#[actix_rt::test] +async fn get_empty_index() { + let mut server = common::Server::with_uid("test"); + let (response, _status) = server.list_indexes().await; + assert!(response.as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn create_and_list_multiple_indices() { + let mut server = common::Server::with_uid("test"); + for i in 0..10 { + server + .create_index(json!({ "uid": format!("test{}", i) })) + .await; + } + let (response, _status) = server.list_indexes().await; + assert_eq!(response.as_array().unwrap().len(), 10); +} + +#[actix_rt::test] +async fn get_unexisting_index_is_error() { + let mut server = common::Server::with_uid("test"); + let (response, status) = server.get_index().await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(response["errorCode"], "index_not_found"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn create_index_twice_is_error() { + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test" })).await; + let (response, status) = server.create_index(json!({ "uid": "test" })).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["errorCode"], "index_already_exists"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn badly_formatted_index_name_is_error() { + let mut server = common::Server::with_uid("$__test"); + let (response, status) = server.create_index(json!({ "uid": "$__test" })).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["errorCode"], "invalid_index_uid"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn correct_response_no_primary_key_index() { + let mut server = common::Server::with_uid("test"); + let (response, _status) = server.create_index(json!({ "uid": "test" })).await; + assert_eq!(response["primaryKey"], Value::Null); +} + +#[actix_rt::test] +async fn correct_response_with_primary_key_index() { + let mut server = common::Server::with_uid("test"); + let (response, _status) = server + .create_index(json!({ "uid": "test", "primaryKey": "test" })) + .await; + assert_eq!(response["primaryKey"], "test"); +} + +#[actix_rt::test] +async fn udpate_unexisting_index_is_error() { + let mut server = common::Server::with_uid("test"); + let (response, status) = server.update_index(json!({ "primaryKey": "foobar" })).await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(response["errorCode"], "index_not_found"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn update_existing_primary_key_is_error() { + let mut server = common::Server::with_uid("test"); + server + .create_index(json!({ "uid": "test", "primaryKey": "key" })) + .await; + let (response, status) = server.update_index(json!({ "primaryKey": "test2" })).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["errorCode"], "primary_key_already_present"); + assert_eq!(response["errorType"], "invalid_request_error"); +} + +#[actix_rt::test] +async fn test_facets_distribution_attribute() { + let mut server = common::Server::test_server().await; + + let (response, _status_code) = server.get_index_stats().await; + + let expected = json!({ + "isIndexing": false, + "numberOfDocuments":77, + "fieldsDistribution":{ + "age":77, + "gender":77, + "phone":77, + "name":77, + "registered":77, + "latitude":77, + "email":77, + "tags":77, + "longitude":77, + "color":77, + "address":77, + "balance":77, + "about":77, + "picture":77, + }, + }); + + assert_json_eq!(expected, response, ordered: true); +} diff --git a/meilisearch-http/_tests/index_update.rs b/meilisearch-http/_tests/index_update.rs new file mode 100644 index 000000000..df4639252 --- /dev/null +++ b/meilisearch-http/_tests/index_update.rs @@ -0,0 +1,200 @@ +use serde_json::json; +use serde_json::Value; +use assert_json_diff::assert_json_include; + +mod common; + +#[actix_rt::test] +async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + let dataset = include_bytes!("assets/test_set.json"); + + let body: Value = serde_json::from_slice(dataset).unwrap(); + + // 2. Index the documents from movies.json, present inside of assets directory + server.add_or_replace_multiple_documents(body).await; + + // 3. Fetch the status of the indexing done above. + let (response, status_code) = server.get_all_updates_status().await; + + // 4. Verify the fetch is successful and indexing status is 'processed' + assert_eq!(status_code, 200); + assert_eq!(response[0]["status"], "processed"); +} + +#[actix_rt::test] +async fn return_error_when_get_update_status_of_unexisting_index() { + let mut server = common::Server::with_uid("test"); + + // 1. Fetch the status of unexisting index. + let (_, status_code) = server.get_all_updates_status().await; + + // 2. Verify the fetch returned 404 + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn return_empty_when_get_update_status_of_empty_index() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + // 2. Fetch the status of empty index. + let (response, status_code) = server.get_all_updates_status().await; + + // 3. Verify the fetch is successful, and no document are returned + assert_eq!(status_code, 200); + assert_eq!(response, json!([])); +} + +#[actix_rt::test] +async fn return_update_status_of_pushed_documents() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + + let bodies = vec![ + json!([{ + "title": "Test", + "comment": "comment test" + }]), + json!([{ + "title": "Test1", + "comment": "comment test1" + }]), + json!([{ + "title": "Test2", + "comment": "comment test2" + }]), + ]; + + let mut update_ids = Vec::new(); + + let url = "/indexes/test/documents?primaryKey=title"; + for body in bodies { + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + update_ids.push(update_id); + } + + // 2. Fetch the status of index. + let (response, status_code) = server.get_all_updates_status().await; + + // 3. Verify the fetch is successful, and updates are returned + + let expected = json!([{ + "type": { + "name": "DocumentsAddition", + "number": 1, + }, + "updateId": update_ids[0] + },{ + "type": { + "name": "DocumentsAddition", + "number": 1, + }, + "updateId": update_ids[1] + },{ + "type": { + "name": "DocumentsAddition", + "number": 1, + }, + "updateId": update_ids[2] + },]); + + assert_eq!(status_code, 200); + assert_json_include!(actual: json!(response), expected: expected); +} + +#[actix_rt::test] +async fn return_error_if_index_does_not_exist() { + let mut server = common::Server::with_uid("test"); + + let (response, status_code) = server.get_update_status(42).await; + + assert_eq!(status_code, 404); + assert_eq!(response["errorCode"], "index_not_found"); +} + +#[actix_rt::test] +async fn return_error_if_update_does_not_exist() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + let (response, status_code) = server.get_update_status(42).await; + + assert_eq!(status_code, 404); + assert_eq!(response["errorCode"], "not_found"); +} + +#[actix_rt::test] +async fn should_return_existing_update() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + }); + + // 1. Create Index + let (response, status_code) = server.create_index(body).await; + assert_eq!(status_code, 201); + assert_eq!(response["primaryKey"], json!(null)); + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/test/documents?primaryKey=title"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 202); + + let update_id = response["updateId"].as_u64().unwrap(); + + let expected = json!({ + "type": { + "name": "DocumentsAddition", + "number": 1, + }, + "updateId": update_id + }); + + let (response, status_code) = server.get_update_status(update_id).await; + + assert_eq!(status_code, 200); + assert_json_include!(actual: json!(response), expected: expected); +} diff --git a/meilisearch-http/_tests/lazy_index_creation.rs b/meilisearch-http/_tests/lazy_index_creation.rs new file mode 100644 index 000000000..6730db82e --- /dev/null +++ b/meilisearch-http/_tests/lazy_index_creation.rs @@ -0,0 +1,446 @@ +use serde_json::json; + +mod common; + +#[actix_rt::test] +async fn create_index_lazy_by_pushing_documents() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Add documents + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/movies/documents?primaryKey=title"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +#[actix_rt::test] +async fn create_index_lazy_by_pushing_documents_and_discover_pk() { + let mut server = common::Server::with_uid("movies"); + + // 1 - Add documents + + let body = json!([{ + "id": 1, + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/movies/documents"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 202); + let update_id = response["updateId"].as_u64().unwrap(); + server.wait_update_id(update_id).await; + + // 3 - Check update success + + let (response, status_code) = server.get_update_status(update_id).await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "processed"); +} + +#[actix_rt::test] +async fn create_index_lazy_by_pushing_documents_with_wrong_name() { + let server = common::Server::with_uid("wrong&name"); + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/wrong&name/documents?primaryKey=title"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_index_uid"); +} + +#[actix_rt::test] +async fn create_index_lazy_add_documents_failed() { + let mut server = common::Server::with_uid("wrong&name"); + + let body = json!([{ + "title": "Test", + "comment": "comment test" + }]); + + let url = "/indexes/wrong&name/documents"; + let (response, status_code) = server.post_request(&url, body).await; + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_index_uid"); + + let (_, status_code) = server.get_index().await; + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_settings() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["name"], + }); + + server.update_all_settings(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_settings_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "other", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "anotherSettings": ["name"], + }); + + let (_, status_code) = server.update_all_settings_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_ranking_rules() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ]); + + server.update_ranking_rules(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_ranking_rules_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!({ + "rankingRules": 123, + }); + + let (_, status_code) = server.update_ranking_rules_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_distinct_attribute() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!("type"); + + server.update_distinct_attribute(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_distinct_attribute_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (resp, status_code) = server.update_distinct_attribute_sync(body.clone()).await; + eprintln!("resp: {:?}", resp); + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (resp, status_code) = server.get_all_settings().await; + eprintln!("resp: {:?}", resp); + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_searchable_attributes() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(["title", "description"]); + + server.update_searchable_attributes(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_searchable_attributes_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server.update_searchable_attributes_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_displayed_attributes() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(["title", "description"]); + + server.update_displayed_attributes(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_displayed_attributes_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server.update_displayed_attributes_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_attributes_for_faceting() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(["title", "description"]); + + server.update_attributes_for_faceting(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_attributes_for_faceting_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server + .update_attributes_for_faceting_sync(body.clone()) + .await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_synonyms() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!({ + "road": ["street", "avenue"], + "street": ["avenue"], + }); + + server.update_synonyms(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_synonyms_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server.update_synonyms_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_stop_words() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(["le", "la", "les"]); + + server.update_stop_words(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 200); +} + +#[actix_rt::test] +async fn create_index_lazy_by_sending_stop_words_with_error() { + let mut server = common::Server::with_uid("movies"); + // 2 - Send the settings + + let body = json!(123); + + let (_, status_code) = server.update_stop_words_sync(body.clone()).await; + assert_eq!(status_code, 400); + + // 3 - Get all settings and compare to the previous one + + let (_, status_code) = server.get_all_settings().await; + + assert_eq!(status_code, 404); +} diff --git a/meilisearch-http/_tests/placeholder_search.rs b/meilisearch-http/_tests/placeholder_search.rs new file mode 100644 index 000000000..048ab7f8b --- /dev/null +++ b/meilisearch-http/_tests/placeholder_search.rs @@ -0,0 +1,629 @@ +use std::convert::Into; + +use serde_json::json; +use serde_json::Value; +use std::cell::RefCell; +use std::sync::Mutex; + +#[macro_use] +mod common; + +#[actix_rt::test] +async fn placeholder_search_with_limit() { + let mut server = common::Server::test_server().await; + + let query = json! ({ + "limit": 3 + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + assert_eq!(response["hits"].as_array().unwrap().len(), 3); + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_offset() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "limit": 6, + }); + + // hack to take a value out of macro (must implement UnwindSafe) + let expected = Mutex::new(RefCell::new(Vec::new())); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + // take results at offset 3 as reference + let lock = expected.lock().unwrap(); + lock.replace(response["hits"].as_array().unwrap()[3..6].to_vec()); + }); + let expected = expected.into_inner().unwrap().into_inner(); + + let query = json!({ + "limit": 3, + "offset": 3, + }); + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + let response = response["hits"].as_array().unwrap(); + assert_eq!(&expected, response); + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_attribute_to_highlight_wildcard() { + // there should be no highlight in placeholder search + let mut server = common::Server::test_server().await; + + let query = json!({ + "limit": 1, + "attributesToHighlight": ["*"] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + let result = response["hits"].as_array().unwrap()[0].as_object().unwrap(); + for value in result.values() { + assert!(value.to_string().find("").is_none()); + } + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_matches() { + // matches is always empty + let mut server = common::Server::test_server().await; + + let query = json!({ + "matches": true + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + let result = response["hits"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_object().unwrap()["_matchesInfo"].clone()) + .all(|m| m.as_object().unwrap().is_empty()); + assert!(result); + }); +} + +#[actix_rt::test] +async fn placeholder_search_witch_crop() { + // placeholder search crop always crop from beggining + let mut server = common::Server::test_server().await; + + let query = json!({ + "attributesToCrop": ["about"], + "cropLength": 20 + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 200); + + let hits = response["hits"].as_array().unwrap(); + + for hit in hits { + let hit = hit.as_object().unwrap(); + let formatted = hit["_formatted"].as_object().unwrap(); + + let about = hit["about"].as_str().unwrap(); + let about_formatted = formatted["about"].as_str().unwrap(); + // the formatted about length should be about 20 characters long + assert!(about_formatted.len() < 20 + 10); + // the formatted part should be located at the beginning of the original one + assert_eq!(about.find(&about_formatted).unwrap(), 0); + } + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_attributes_to_retrieve() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "limit": 1, + "attributesToRetrieve": ["gender", "about"], + }); + + test_post_get_search!(server, query, |response, _status_code| { + let hit = response["hits"].as_array().unwrap()[0].as_object().unwrap(); + assert_eq!(hit.values().count(), 2); + let _ = hit["gender"]; + let _ = hit["about"]; + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_filter() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "filters": "color='green'" + }); + + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green")); + }); + + let query = json!({ + "filters": "tags=bug" + }); + + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + let value = Value::String(String::from("bug")); + assert!(hits + .iter() + .all(|v| v["tags"].as_array().unwrap().contains(&value))); + }); + + let query = json!({ + "filters": "color='green' AND (tags='bug' OR tags='wontfix')" + }); + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + let bug = Value::String(String::from("bug")); + let wontfix = Value::String(String::from("wontfix")); + assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green" + && v["tags"].as_array().unwrap().contains(&bug) + || v["tags"].as_array().unwrap().contains(&wontfix))); + }); +} + +#[actix_rt::test] +async fn placeholder_test_faceted_search_valid() { + let mut server = common::Server::test_server().await; + + // simple tests on attributes with string value + let body = json!({ + "attributesForFaceting": ["color"] + }); + + server.update_all_settings(body).await; + + let query = json!({ + "facetFilters": ["color:green"] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "Green")); + }); + + let query = json!({ + "facetFilters": [["color:blue"]] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue")); + }); + + let query = json!({ + "facetFilters": ["color:Blue"] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue")); + }); + + // test on arrays: ["tags:bug"] + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + + server.update_all_settings(body).await; + + let query = json!({ + "facetFilters": ["tags:bug"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value + .get("tags") + .unwrap() + .as_array() + .unwrap() + .contains(&Value::String("bug".to_owned())))); + }); + + // test and: ["color:blue", "tags:bug"] + let query = json!({ + "facetFilters": ["color:blue", "tags:bug"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue" + && value + .get("tags") + .unwrap() + .as_array() + .unwrap() + .contains(&Value::String("bug".to_owned())))); + }); + + // test or: [["color:blue", "color:green"]] + let query = json!({ + "facetFilters": [["color:blue", "color:green"]] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue" + || value.get("color").unwrap() == "Green")); + }); + // test and-or: ["tags:bug", ["color:blue", "color:green"]] + let query = json!({ + "facetFilters": ["tags:bug", ["color:blue", "color:green"]] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value + .get("tags") + .unwrap() + .as_array() + .unwrap() + .contains(&Value::String("bug".to_owned())) + && (value.get("color").unwrap() == "blue" + || value.get("color").unwrap() == "Green"))); + }); +} + +#[actix_rt::test] +async fn placeholder_test_faceted_search_invalid() { + let mut server = common::Server::test_server().await; + + //no faceted attributes set + let query = json!({ + "facetFilters": ["color:blue"] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + server.update_all_settings(body).await; + // empty arrays are error + // [] + let query = json!({ + "facetFilters": [] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + // [[]] + let query = json!({ + "facetFilters": [[]] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + // ["color:green", []] + let query = json!({ + "facetFilters": ["color:green", []] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + + // too much depth + // [[[]]] + let query = json!({ + "facetFilters": [[[]]] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + // [["color:green", ["color:blue"]]] + let query = json!({ + "facetFilters": [["color:green", ["color:blue"]]] + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); + // "color:green" + let query = json!({ + "facetFilters": "color:green" + }); + test_post_get_search!(server, query, |_response, status_code| assert_ne!( + status_code, + 202 + )); +} + +#[actix_rt::test] +async fn placeholder_test_facet_count() { + let mut server = common::Server::test_server().await; + + // test without facet distribution + let query = json!({}); + test_post_get_search!(server, query, |response, _status_code| { + assert!(response.get("exhaustiveFacetsCount").is_none()); + assert!(response.get("facetsDistribution").is_none()); + }); + + // test no facets set, search on color + let query = json!({ + "facetsDistribution": ["color"] + }); + test_post_get_search!(server, query.clone(), |_response, status_code| { + assert_eq!(status_code, 400); + }); + + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + server.update_all_settings(body).await; + // same as before, but now facets are set: + test_post_get_search!(server, query, |response, _status_code| { + println!("{}", response); + assert!(response.get("exhaustiveFacetsCount").is_some()); + assert_eq!( + response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 1 + ); + }); + // searching on color and tags + let query = json!({ + "facetsDistribution": ["color", "tags"] + }); + test_post_get_search!(server, query, |response, _status_code| { + let facets = response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap(); + assert_eq!(facets.values().count(), 2); + assert_ne!( + !facets + .get("color") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 0 + ); + assert_ne!( + !facets + .get("tags") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 0 + ); + }); + // wildcard + let query = json!({ + "facetsDistribution": ["*"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_eq!( + response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 2 + ); + }); + // wildcard with other attributes: + let query = json!({ + "facetsDistribution": ["color", "*"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_eq!( + response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 2 + ); + }); + + // empty facet list + let query = json!({ + "facetsDistribution": [] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_eq!( + response + .get("facetsDistribution") + .unwrap() + .as_object() + .unwrap() + .values() + .count(), + 0 + ); + }); + + // attr not set as facet passed: + let query = json!({ + "facetsDistribution": ["gender"] + }); + test_post_get_search!(server, query, |_response, status_code| { + assert_eq!(status_code, 400); + }); +} + +#[actix_rt::test] +#[should_panic] +async fn placeholder_test_bad_facet_distribution() { + let mut server = common::Server::test_server().await; + // string instead of array: + let query = json!({ + "facetsDistribution": "color" + }); + test_post_get_search!(server, query, |_response, _status_code| {}); + + // invalid value in array: + let query = json!({ + "facetsDistribution": ["color", true] + }); + test_post_get_search!(server, query, |_response, _status_code| {}); +} + +#[actix_rt::test] +async fn placeholder_test_sort() { + let mut server = common::Server::test_server().await; + + let body = json!({ + "rankingRules": ["asc(age)"], + "attributesForFaceting": ["color"] + }); + server.update_all_settings(body).await; + let query = json!({}); + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + hits.iter() + .map(|v| v["age"].as_u64().unwrap()) + .fold(0, |prev, cur| { + assert!(cur >= prev); + cur + }); + }); + + let query = json!({ + "facetFilters": ["color:green"] + }); + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + hits.iter() + .map(|v| v["age"].as_u64().unwrap()) + .fold(0, |prev, cur| { + assert!(cur >= prev); + cur + }); + }); +} + +#[actix_rt::test] +async fn placeholder_search_with_empty_query() { + let mut server = common::Server::test_server().await; + + let query = json! ({ + "q": "", + "limit": 3 + }); + + test_post_get_search!(server, query, |response, status_code| { + eprintln!("{}", response); + assert_eq!(status_code, 200); + assert_eq!(response["hits"].as_array().unwrap().len(), 3); + }); +} + +#[actix_rt::test] +async fn test_filter_nb_hits_search_placeholder() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + let documents = json!([ + { + "id": 1, + "content": "a", + "color": "green", + "size": 1, + }, + { + "id": 2, + "content": "a", + "color": "green", + "size": 2, + }, + { + "id": 3, + "content": "a", + "color": "blue", + "size": 3, + }, + ]); + + server.add_or_update_multiple_documents(documents).await; + let (response, _) = server.search_post(json!({})).await; + assert_eq!(response["nbHits"], 3); + + server.update_distinct_attribute(json!("color")).await; + + let (response, _) = server.search_post(json!({})).await; + assert_eq!(response["nbHits"], 2); + + let (response, _) = server.search_post(json!({"filters": "size < 3"})).await; + println!("result: {}", response); + assert_eq!(response["nbHits"], 1); +} diff --git a/meilisearch-http/_tests/search.rs b/meilisearch-http/_tests/search.rs new file mode 100644 index 000000000..267c98265 --- /dev/null +++ b/meilisearch-http/_tests/search.rs @@ -0,0 +1,1879 @@ +use std::convert::Into; + +use assert_json_diff::assert_json_eq; +use serde_json::json; +use serde_json::Value; + +#[macro_use] mod common; + +#[actix_rt::test] +async fn search() { + let mut server = common::Server::test_server().await; + + let query = json! ({ + "q": "exercitation" + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + }, + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 49, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ], + "isActive": true + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + let hits = response["hits"].as_array().unwrap(); + let hits: Vec = hits.iter().cloned().take(3).collect(); + assert_json_eq!(expected.clone(), serde_json::to_value(hits).unwrap(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_no_params() { + let mut server = common::Server::test_server().await; + + let query = json! ({}); + + // an empty search should return the 20 first indexed document + let dataset: Vec = serde_json::from_slice(include_bytes!("assets/test_set.json")).unwrap(); + let expected: Vec = dataset.into_iter().take(20).collect(); + let expected: Value = serde_json::to_value(expected).unwrap(); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_in_unexisting_index() { + let mut server = common::Server::with_uid("test"); + + let query = json! ({ + "q": "exercitation" + }); + + let expected = json! ({ + "message": "Index test not found", + "errorCode": "index_not_found", + "errorType": "invalid_request_error", + "errorLink": "https://docs.meilisearch.com/errors#index_not_found" + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(404, status_code); + assert_json_eq!(expected.clone(), response.clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_unexpected_params() { + + let query = json! ({"lol": "unexpected"}); + + let expected = "unknown field `lol`, expected one of `q`, `offset`, `limit`, `attributesToRetrieve`, `attributesToCrop`, `cropLength`, `attributesToHighlight`, `filters`, `matches`, `facetFilters`, `facetsDistribution` at line 1 column 6"; + + let post_query = serde_json::from_str::(&query.to_string()); + assert!(post_query.is_err()); + assert_eq!(expected, post_query.err().unwrap().to_string()); + + let get_query: Result = serde_json::from_str(&query.to_string()); + assert!(get_query.is_err()); + assert_eq!(expected, get_query.err().unwrap().to_string()); +} + +#[actix_rt::test] +async fn search_with_limit() { + let mut server = common::Server::test_server().await; + + let query = json! ({ + "q": "exercitation", + "limit": 3 + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + }, + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 49, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ], + "isActive": true + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_offset() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 3, + "offset": 1 + }); + + let expected = json!([ + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 49, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ], + "isActive": true + }, + { + "id": 0, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ], + "isActive": false + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attribute_to_highlight_wildcard() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToHighlight": ["*"] + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attribute_to_highlight_1() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToHighlight": ["name"] + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_matches() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "matches": true + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_matchesInfo": { + "name": [ + { + "start": 0, + "length": 6 + } + ], + "email": [ + { + "start": 0, + "length": 6 + } + ] + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_crop() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 1, + "attributesToCrop": ["about"], + "cropLength": 20 + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attributes_to_retrieve() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","color","gender"], + }); + + let expected = json!([ + { + "name": "Cherry Orr", + "age": 27, + "color": "Green", + "gender": "female" + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attributes_to_retrieve_wildcard() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["*"], + }); + + let expected = json!([ + { + "id": 1, + "isActive": true, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ] + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_filter() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 3, + "filters": "gender='male'" + }); + + let expected = json!([ + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 0, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ], + "isActive": false + }, + { + "id": 66, + "balance": "$1,061.49", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "brown", + "name": "Higgins Aguilar", + "gender": "male", + "email": "higginsaguilar@chorizon.com", + "phone": "+1 (911) 540-3791", + "address": "132 Sackman Street, Layhill, Guam, 8729", + "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", + "registered": "2015-04-05T02:10:07 -02:00", + "latitude": 74.702813, + "longitude": 151.314972, + "tags": [ + "bug" + ], + "isActive": true + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 0, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ], + "isActive": false + } + ]); + + let query = json!({ + "q": "exercitation", + "limit": 3, + "filters": "name='Lucas Hess'" + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 2, + "balance": "$2,467.47", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", + "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", + "registered": "2014-10-28T12:59:30 -01:00", + "latitude": -64.008555, + "longitude": 11.867098, + "tags": [ + "good first issue" + ], + "isActive": true + }, + { + "id": 75, + "balance": "$1,913.42", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Emma Jacobs", + "gender": "female", + "email": "emmajacobs@chorizon.com", + "phone": "+1 (899) 554-3847", + "address": "173 Tapscott Street, Esmont, Maine, 7450", + "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", + "registered": "2019-03-29T06:24:13 -01:00", + "latitude": -35.53722, + "longitude": 155.703874, + "tags": [], + "isActive": false + } + ]); + let query = json!({ + "q": "exercitation", + "limit": 3, + "filters": "gender='female' AND (name='Patricia Goff' OR name='Emma Jacobs')" + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 30, + "balance": "$2,021.11", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Stacy Espinoza", + "gender": "female", + "email": "stacyespinoza@chorizon.com", + "phone": "+1 (999) 487-3253", + "address": "931 Alabama Avenue, Bangor, Alaska, 8215", + "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", + "registered": "2014-07-16T06:15:53 -02:00", + "latitude": 41.560197, + "longitude": 177.697, + "tags": [ + "new issue", + "new issue", + "bug" + ], + "isActive": true + }, + { + "id": 31, + "balance": "$3,609.82", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Vilma Garza", + "gender": "female", + "email": "vilmagarza@chorizon.com", + "phone": "+1 (944) 585-2021", + "address": "565 Tech Place, Sedley, Puerto Rico, 858", + "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", + "registered": "2017-06-30T07:43:52 -02:00", + "latitude": -12.574889, + "longitude": -54.771186, + "tags": [ + "new issue", + "wontfix", + "wontfix" + ], + "isActive": false + }, + { + "id": 2, + "balance": "$2,467.47", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", + "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", + "registered": "2014-10-28T12:59:30 -01:00", + "latitude": -64.008555, + "longitude": 11.867098, + "tags": [ + "good first issue" + ], + "isActive": true + } + ]); + let query = json!({ + "q": "exerciatation", + "limit": 3, + "filters": "gender='female' AND (name='Patricia Goff' OR age > 30)" + }); + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 59, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ], + "isActive": true + }, + { + "id": 0, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ], + "isActive": false + }, + { + "id": 66, + "balance": "$1,061.49", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "brown", + "name": "Higgins Aguilar", + "gender": "male", + "email": "higginsaguilar@chorizon.com", + "phone": "+1 (911) 540-3791", + "address": "132 Sackman Street, Layhill, Guam, 8729", + "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", + "registered": "2015-04-05T02:10:07 -02:00", + "latitude": 74.702813, + "longitude": 151.314972, + "tags": [ + "bug" + ], + "isActive": true + } + ]); + let query = json!({ + "q": "exerciatation", + "limit": 3, + "filters": "NOT gender = 'female' AND age > 30" + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); + + let expected = json!([ + { + "id": 11, + "balance": "$1,351.43", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Evans Wagner", + "gender": "male", + "email": "evanswagner@chorizon.com", + "phone": "+1 (889) 496-2332", + "address": "118 Monaco Place, Lutsen, Delaware, 6209", + "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", + "registered": "2016-10-27T01:26:31 -02:00", + "latitude": -77.673222, + "longitude": -142.657214, + "tags": [ + "good first issue", + "good first issue" + ], + "isActive": true + } + ]); + let query = json!({ + "q": "exerciatation", + "filters": "NOT gender = 'female' AND name='Evans Wagner'" + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attributes_to_highlight_and_matches() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToHighlight": ["name","email"], + "matches": true, + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + }, + "_matchesInfo": { + "email": [ + { + "start": 0, + "length": 6 + } + ], + "name": [ + { + "start": 0, + "length": 6 + } + ] + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_attributes_to_highlight_and_matches_and_crop() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exerciatation", + "limit": 1, + "attributesToCrop": ["about"], + "cropLength": 20, + "attributesToHighlight": ["about"], + "matches": true, + }); + + let expected = json!([ + { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true, + "_formatted": { + "id": 1, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ], + "isActive": true + }, + "_matchesInfo": { + "about": [ + { + "start": 0, + "length": 12 + } + ] + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","gender","email"], + "attributesToHighlight": ["name"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "name": "Cherry Orr" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_2() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 1, + "attributesToRetrieve": ["name","age","gender"], + "attributesToCrop": ["about"], + "cropLength": 20, + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "_formatted": { + "about": "Exercitation officia" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_3() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "exercitation", + "limit": 1, + "attributesToRetrieve": ["name","age","gender"], + "attributesToCrop": ["about:20"], + }); + + let expected = json!( [ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "_formatted": { + "about": "Exercitation officia" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_4() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","email","gender"], + "attributesToCrop": ["name:0","email:6"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "name": "Cherry", + "email": "cherryorr" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_5() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","email","gender"], + "attributesToCrop": ["*","email:6"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "name": "Cherry Orr", + "email": "cherryorr", + "age": 27, + "gender": "female" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_6() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","email","gender"], + "attributesToCrop": ["*","email:10"], + "attributesToHighlight": ["name"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_7() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","gender","email"], + "attributesToCrop": ["*","email:6"], + "attributesToHighlight": ["*"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "_formatted": { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn search_with_differents_attributes_8() { + let mut server = common::Server::test_server().await; + + let query = json!({ + "q": "cherry", + "limit": 1, + "attributesToRetrieve": ["name","age","email","gender","address"], + "attributesToCrop": ["*","email:6"], + "attributesToHighlight": ["*","address"], + }); + + let expected = json!([ + { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "_formatted": { + "age": 27, + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr", + "address": "442 Beverly Road, Ventress, New Mexico, 3361" + } + } + ]); + + test_post_get_search!(server, query, |response, _status_code| { + assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); + }); +} + +#[actix_rt::test] +async fn test_faceted_search_valid() { + // set facetting attributes before adding documents + let mut server = common::Server::with_uid("test"); + server.create_index(json!({ "uid": "test" })).await; + + let body = json!({ + "attributesForFaceting": ["color"] + }); + server.update_all_settings(body).await; + + let dataset = include_bytes!("assets/test_set.json"); + let body: Value = serde_json::from_slice(dataset).unwrap(); + server.add_or_update_multiple_documents(body).await; + + // simple tests on attributes with string value + + let query = json!({ + "q": "a", + "facetFilters": ["color:green"] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "Green")); + }); + + let query = json!({ + "q": "a", + "facetFilters": [["color:blue"]] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue")); + }); + + let query = json!({ + "q": "a", + "facetFilters": ["color:Blue"] + }); + + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("color").unwrap() == "blue")); + }); + + // test on arrays: ["tags:bug"] + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + + server.update_all_settings(body).await; + + let query = json!({ + "q": "a", + "facetFilters": ["tags:bug"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); + }); + + // test and: ["color:blue", "tags:bug"] + let query = json!({ + "q": "a", + "facetFilters": ["color:blue", "tags:bug"] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| value + .get("color") + .unwrap() == "blue" + && value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); + }); + + // test or: [["color:blue", "color:green"]] + let query = json!({ + "q": "a", + "facetFilters": [["color:blue", "color:green"]] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| + value + .get("color") + .unwrap() == "blue" + || value + .get("color") + .unwrap() == "Green")); + }); + // test and-or: ["tags:bug", ["color:blue", "color:green"]] + let query = json!({ + "q": "a", + "facetFilters": ["tags:bug", ["color:blue", "color:green"]] + }); + test_post_get_search!(server, query, |response, _status_code| { + assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); + assert!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .iter() + .all(|value| + value + .get("tags") + .unwrap() + .as_array() + .unwrap() + .contains(&Value::String("bug".to_owned())) + && (value + .get("color") + .unwrap() == "blue" + || value + .get("color") + .unwrap() == "Green"))); + + }); +} + +#[actix_rt::test] +async fn test_faceted_search_invalid() { + let mut server = common::Server::test_server().await; + + //no faceted attributes set + let query = json!({ + "q": "a", + "facetFilters": ["color:blue"] + }); + + test_post_get_search!(server, query, |response, status_code| { + + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + server.update_all_settings(body).await; + // empty arrays are error + // [] + let query = json!({ + "q": "a", + "facetFilters": [] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + // [[]] + let query = json!({ + "q": "a", + "facetFilters": [[]] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + // ["color:green", []] + let query = json!({ + "q": "a", + "facetFilters": ["color:green", []] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + // too much depth + // [[[]]] + let query = json!({ + "q": "a", + "facetFilters": [[[]]] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + // [["color:green", ["color:blue"]]] + let query = json!({ + "q": "a", + "facetFilters": [["color:green", ["color:blue"]]] + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); + + // "color:green" + let query = json!({ + "q": "a", + "facetFilters": "color:green" + }); + + test_post_get_search!(server, query, |response, status_code| { + assert_eq!(status_code, 400); + assert_eq!(response["errorCode"], "invalid_facet"); + }); +} + +#[actix_rt::test] +async fn test_facet_count() { + let mut server = common::Server::test_server().await; + + // test without facet distribution + let query = json!({ + "q": "a", + }); + test_post_get_search!(server, query, |response, _status_code|{ + assert!(response.get("exhaustiveFacetsCount").is_none()); + assert!(response.get("facetsDistribution").is_none()); + }); + + // test no facets set, search on color + let query = json!({ + "q": "a", + "facetsDistribution": ["color"] + }); + test_post_get_search!(server, query.clone(), |_response, status_code|{ + assert_eq!(status_code, 400); + }); + + let body = json!({ + "attributesForFaceting": ["color", "tags"] + }); + server.update_all_settings(body).await; + // same as before, but now facets are set: + test_post_get_search!(server, query, |response, _status_code|{ + println!("{}", response); + assert!(response.get("exhaustiveFacetsCount").is_some()); + assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 1); + // assert that case is preserved + assert!(response["facetsDistribution"] + .as_object() + .unwrap()["color"] + .as_object() + .unwrap() + .get("Green") + .is_some()); + }); + // searching on color and tags + let query = json!({ + "q": "a", + "facetsDistribution": ["color", "tags"] + }); + test_post_get_search!(server, query, |response, _status_code|{ + let facets = response.get("facetsDistribution").unwrap().as_object().unwrap(); + assert_eq!(facets.values().count(), 2); + assert_ne!(!facets.get("color").unwrap().as_object().unwrap().values().count(), 0); + assert_ne!(!facets.get("tags").unwrap().as_object().unwrap().values().count(), 0); + }); + // wildcard + let query = json!({ + "q": "a", + "facetsDistribution": ["*"] + }); + test_post_get_search!(server, query, |response, _status_code|{ + assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); + }); + // wildcard with other attributes: + let query = json!({ + "q": "a", + "facetsDistribution": ["color", "*"] + }); + test_post_get_search!(server, query, |response, _status_code|{ + assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); + }); + + // empty facet list + let query = json!({ + "q": "a", + "facetsDistribution": [] + }); + test_post_get_search!(server, query, |response, _status_code|{ + assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 0); + }); + + // attr not set as facet passed: + let query = json!({ + "q": "a", + "facetsDistribution": ["gender"] + }); + test_post_get_search!(server, query, |_response, status_code|{ + assert_eq!(status_code, 400); + }); + +} + +#[actix_rt::test] +#[should_panic] +async fn test_bad_facet_distribution() { + let mut server = common::Server::test_server().await; + // string instead of array: + let query = json!({ + "q": "a", + "facetsDistribution": "color" + }); + test_post_get_search!(server, query, |_response, _status_code| {}); + + // invalid value in array: + let query = json!({ + "q": "a", + "facetsDistribution": ["color", true] + }); + test_post_get_search!(server, query, |_response, _status_code| {}); +} + +#[actix_rt::test] +async fn highlight_cropped_text() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + let doc = json!([ + { + "id": 1, + "body": r##"well, it may not work like that, try the following: +1. insert your trip +2. google your `searchQuery` +3. find a solution +> say hello"## + } + ]); + server.add_or_replace_multiple_documents(doc).await; + + // tests from #680 + //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=30"; + let query = json!({ + "q": "insert", + "attributesToHighlight": ["*"], + "attributesToCrop": ["body"], + "cropLength": 30, + }); + let expected_response = "that, try the following: \n1. insert your trip\n2. google your"; + test_post_get_search!(server, query, |response, _status_code|{ + assert_eq!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .get(0) + .unwrap() + .as_object() + .unwrap() + .get("_formatted") + .unwrap() + .as_object() + .unwrap() + .get("body") + .unwrap() + , &Value::String(expected_response.to_owned())); + }); + + //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=80"; + let query = json!({ + "q": "insert", + "attributesToHighlight": ["*"], + "attributesToCrop": ["body"], + "cropLength": 80, + }); + let expected_response = "well, it may not work like that, try the following: \n1. insert your trip\n2. google your `searchQuery`\n3. find a solution \n> say hello"; + test_post_get_search!(server, query, |response, _status_code| { + assert_eq!(response + .get("hits") + .unwrap() + .as_array() + .unwrap() + .get(0) + .unwrap() + .as_object() + .unwrap() + .get("_formatted") + .unwrap() + .as_object() + .unwrap() + .get("body") + .unwrap() + , &Value::String(expected_response.to_owned())); + }); +} + +#[actix_rt::test] +async fn well_formated_error_with_bad_request_params() { + let mut server = common::Server::with_uid("test"); + let query = "foo=bar"; + let (response, _status_code) = server.search_get(query).await; + assert!(response.get("message").is_some()); + assert!(response.get("errorCode").is_some()); + assert!(response.get("errorType").is_some()); + assert!(response.get("errorLink").is_some()); +} + + +#[actix_rt::test] +async fn update_documents_with_facet_distribution() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + let settings = json!({ + "attributesForFaceting": ["genre"], + "displayedAttributes": ["genre"], + "searchableAttributes": ["genre"] + }); + server.update_all_settings(settings).await; + let update1 = json!([ + { + "id": "1", + "type": "album", + "title": "Nevermind", + "genre": ["grunge", "alternative"] + }, + { + "id": "2", + "type": "album", + "title": "Mellon Collie and the Infinite Sadness", + "genre": ["alternative", "rock"] + }, + { + "id": "3", + "type": "album", + "title": "The Queen Is Dead", + "genre": ["indie", "rock"] + } + ]); + server.add_or_update_multiple_documents(update1).await; + let search = json!({ + "q": "album", + "facetsDistribution": ["genre"] + }); + let (response1, _) = server.search_post(search.clone()).await; + let expected_facet_distribution = json!({ + "genre": { + "grunge": 1, + "alternative": 2, + "rock": 2, + "indie": 1 + } + }); + assert_json_eq!(expected_facet_distribution.clone(), response1["facetsDistribution"].clone()); + + let update2 = json!([ + { + "id": "3", + "title": "The Queen Is Very Dead" + } + ]); + server.add_or_update_multiple_documents(update2).await; + let (response2, _) = server.search_post(search).await; + assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone()); +} + +#[actix_rt::test] +async fn test_filter_nb_hits_search_normal() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + let documents = json!([ + { + "id": 1, + "content": "a", + "color": "green", + "size": 1, + }, + { + "id": 2, + "content": "a", + "color": "green", + "size": 2, + }, + { + "id": 3, + "content": "a", + "color": "blue", + "size": 3, + }, + ]); + + server.add_or_update_multiple_documents(documents).await; + let (response, _) = server.search_post(json!({"q": "a"})).await; + assert_eq!(response["nbHits"], 3); + + let (response, _) = server.search_post(json!({"q": "a", "filters": "size = 1"})).await; + assert_eq!(response["nbHits"], 1); + + server.update_distinct_attribute(json!("color")).await; + + let (response, _) = server.search_post(json!({"q": "a"})).await; + assert_eq!(response["nbHits"], 2); + + let (response, _) = server.search_post(json!({"q": "a", "filters": "size < 3"})).await; + println!("result: {}", response); + assert_eq!(response["nbHits"], 1); +} diff --git a/meilisearch-http/_tests/search_settings.rs b/meilisearch-http/_tests/search_settings.rs new file mode 100644 index 000000000..46417498d --- /dev/null +++ b/meilisearch-http/_tests/search_settings.rs @@ -0,0 +1,538 @@ +use assert_json_diff::assert_json_eq; +use serde_json::json; +use std::convert::Into; + +mod common; + +#[actix_rt::test] +async fn search_with_settings_basic() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "age", + "color", + "gender", + "email", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": null, + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=ea%20exercitation&limit=3"; + + let expect = json!([ + { + "balance": "$2,467.47", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700" + }, + { + "balance": "$3,344.40", + "age": 35, + "color": "blue", + "name": "Adeline Flynn", + "gender": "female", + "email": "adelineflynn@chorizon.com", + "phone": "+1 (994) 600-2840", + "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948" + }, + { + "balance": "$3,394.96", + "age": 25, + "color": "blue", + "name": "Aida Kirby", + "gender": "female", + "email": "aidakirby@chorizon.com", + "phone": "+1 (942) 532-2325", + "address": "797 Engert Avenue, Wilsonia, Idaho, 6532" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_stop_words() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "age", + "color", + "gender", + "email", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": ["ea"], + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=ea%20exercitation&limit=3"; + let expect = json!([ + { + "balance": "$1,921.58", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219" + }, + { + "balance": "$1,706.13", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361" + }, + { + "balance": "$1,476.39", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_synonyms() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "age", + "color", + "gender", + "email", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": null, + "synonyms": { + "application": [ + "exercitation" + ] + }, + }); + + server.update_all_settings(config).await; + + let query = "q=application&limit=3"; + let expect = json!([ + { + "balance": "$1,921.58", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219" + }, + { + "balance": "$1,706.13", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361" + }, + { + "balance": "$1,476.39", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_ranking_rules() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "age", + "color", + "gender", + "email", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": null, + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=exarcitation&limit=3"; + let expect = json!([ + { + "balance": "$1,921.58", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219" + }, + { + "balance": "$1,706.13", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361" + }, + { + "balance": "$1,476.39", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + println!("{}", response["hits"].clone()); + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_searchable_attributes() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "age", + "color", + "gender", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone", + "address", + "balance" + ], + "stopWords": null, + "synonyms": { + "exarcitation": [ + "exercitation" + ] + }, + }); + + server.update_all_settings(config).await; + + let query = "q=Carol&limit=3"; + let expect = json!([ + { + "balance": "$1,440.09", + "age": 40, + "color": "blue", + "name": "Levy Whitley", + "gender": "male", + "email": "levywhitley@chorizon.com", + "phone": "+1 (911) 458-2411", + "address": "187 Thomas Street, Hachita, North Carolina, 2989" + }, + { + "balance": "$1,977.66", + "age": 36, + "color": "brown", + "name": "Combs Stanley", + "gender": "male", + "email": "combsstanley@chorizon.com", + "phone": "+1 (827) 419-2053", + "address": "153 Beverley Road, Siglerville, South Carolina, 3666" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_displayed_attributes() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "age", + "color", + "gender", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender", + "color", + "email", + "phone" + ], + "stopWords": null, + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=exercitation&limit=3"; + let expect = json!([ + { + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243" + }, + { + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174" + }, + { + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +#[actix_rt::test] +async fn search_with_settings_searchable_attributes_2() { + let mut server = common::Server::test_server().await; + + let config = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "desc(age)", + "exactness", + "desc(balance)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "age", + "color", + "gender", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "age", + "gender" + ], + "stopWords": null, + "synonyms": null, + }); + + server.update_all_settings(config).await; + + let query = "q=exercitation&limit=3"; + let expect = json!([ + { + "age": 31, + "name": "Harper Carson", + "gender": "male" + }, + { + "age": 27, + "name": "Cherry Orr", + "gender": "female" + }, + { + "age": 28, + "name": "Maureen Dale", + "gender": "female" + } + ]); + + let (response, _status_code) = server.search_get(query).await; + assert_json_eq!(expect, response["hits"].clone(), ordered: false); +} + +// issue #798 +#[actix_rt::test] +async fn distinct_attributes_returns_name_not_id() { + let mut server = common::Server::test_server().await; + let settings = json!({ + "distinctAttribute": "color", + }); + server.update_all_settings(settings).await; + let (response, _) = server.get_all_settings().await; + assert_eq!(response["distinctAttribute"], "color"); + let (response, _) = server.get_distinct_attribute().await; + assert_eq!(response, "color"); +} diff --git a/meilisearch-http/_tests/settings.rs b/meilisearch-http/_tests/settings.rs new file mode 100644 index 000000000..6b125c13a --- /dev/null +++ b/meilisearch-http/_tests/settings.rs @@ -0,0 +1,523 @@ +use assert_json_diff::assert_json_eq; +use serde_json::json; +use std::convert::Into; +mod common; + +#[actix_rt::test] +async fn write_all_and_delete() { + let mut server = common::Server::test_server().await; + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["name"], + }); + + server.update_all_settings(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: false); + + // 4 - Delete all settings + + server.delete_all_settings().await; + + // 5 - Get all settings and check if they are set to default values + + let (response, _status_code) = server.get_all_settings().await; + + let expect = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": ["*"], + "displayedAttributes": ["*"], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [], + }); + + assert_json_eq!(expect, response, ordered: false); +} + +#[actix_rt::test] +async fn write_all_and_update() { + let mut server = common::Server::test_server().await; + + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["name"], + }); + + server.update_all_settings(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: false); + + // 4 - Update all settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(age)", + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "color", + "age", + ], + "displayedAttributes": [ + "name", + "color", + "age", + "registered", + "picture", + ], + "stopWords": [], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["title"], + }); + + server.update_all_settings(body).await; + + // 5 - Get all settings and check if the content is the same of (4) + + let (response, _status_code) = server.get_all_settings().await; + + let expected = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(age)", + ], + "distinctAttribute": null, + "searchableAttributes": [ + "name", + "color", + "age", + ], + "displayedAttributes": [ + "name", + "color", + "age", + "registered", + "picture", + ], + "stopWords": [], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": ["title"], + }); + + assert_json_eq!(expected, response, ordered: false); +} + +#[actix_rt::test] +async fn test_default_settings() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + }); + server.create_index(body).await; + + // 1 - Get all settings and compare to the previous one + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": ["*"], + "displayedAttributes": ["*"], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [], + }); + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: false); +} + +#[actix_rt::test] +async fn test_default_settings_2() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + // 1 - Get all settings and compare to the previous one + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": ["*"], + "displayedAttributes": ["*"], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [], + }); + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: false); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/516 +#[actix_rt::test] +async fn write_setting_and_update_partial() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + }); + server.create_index(body).await; + + // 2 - Send the settings + + let body = json!({ + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ] + }); + + server.update_all_settings(body.clone()).await; + + // 2 - Send the settings + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(age)", + "desc(registered)", + ], + "distinctAttribute": "id", + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + }); + + server.update_all_settings(body.clone()).await; + + // 2 - Send the settings + + let expected = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(age)", + "desc(registered)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "about" + ], + "displayedAttributes": [ + "name", + "gender", + "email", + "registered", + "age", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["street", "avenue"], + "street": ["avenue"], + }, + "attributesForFaceting": [], + }); + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(expected, response, ordered: false); +} + +#[actix_rt::test] +async fn attributes_for_faceting_settings() { + let mut server = common::Server::test_server().await; + // initial attributes array should be empty + let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; + assert_eq!(response, json!([])); + // add an attribute and test for its presence + let (_response, _status_code) = server.post_request_async( + "/indexes/test/settings/attributes-for-faceting", + json!(["foobar"])).await; + let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; + assert_eq!(response, json!(["foobar"])); + // remove all attributes and test for emptiness + let (_response, _status_code) = server.delete_request_async( + "/indexes/test/settings/attributes-for-faceting").await; + let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; + assert_eq!(response, json!([])); +} + +#[actix_rt::test] +async fn setting_ranking_rules_dont_mess_with_other_settings() { + let mut server = common::Server::test_server().await; + let body = json!({ + "rankingRules": ["asc(foobar)"] + }); + server.update_all_settings(body).await; + let (response, _) = server.get_all_settings().await; + assert_eq!(response["rankingRules"].as_array().unwrap().len(), 1); + assert_eq!(response["rankingRules"].as_array().unwrap().first().unwrap().as_str().unwrap(), "asc(foobar)"); + assert!(!response["searchableAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); + assert!(!response["displayedAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); +} + +#[actix_rt::test] +async fn displayed_and_searchable_attributes_reset_to_wildcard() { + let mut server = common::Server::test_server().await; + server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; + let (response, _) = server.get_all_settings().await; + + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); + + server.delete_searchable_attributes().await; + server.delete_displayed_attributes().await; + + let (response, _) = server.get_all_settings().await; + + assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); + + let mut server = common::Server::test_server().await; + server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; + let (response, _) = server.get_all_settings().await; + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); + + server.update_all_settings(json!({ "searchableAttributes": [], "displayedAttributes": [] })).await; + + let (response, _) = server.get_all_settings().await; + + assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); +} + +#[actix_rt::test] +async fn settings_that_contains_wildcard_is_wildcard() { + let mut server = common::Server::test_server().await; + server.update_all_settings(json!({ "searchableAttributes": ["color", "*"], "displayedAttributes": ["color", "*"] })).await; + + let (response, _) = server.get_all_settings().await; + + assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); + assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); + assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); +} + +#[actix_rt::test] +async fn test_displayed_attributes_field() { + let mut server = common::Server::test_server().await; + + let body = json!({ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ], + "distinctAttribute": "id", + "searchableAttributes": [ + "id", + "name", + "color", + "gender", + "email", + "phone", + "address", + "registered", + "about" + ], + "displayedAttributes": [ + "age", + "email", + "gender", + "name", + "registered", + ], + "stopWords": [ + "ad", + "in", + "ut", + ], + "synonyms": { + "road": ["avenue", "street"], + "street": ["avenue"], + }, + "attributesForFaceting": ["name"], + }); + + server.update_all_settings(body.clone()).await; + + let (response, _status_code) = server.get_all_settings().await; + + assert_json_eq!(body, response, ordered: true); +} \ No newline at end of file diff --git a/meilisearch-http/_tests/settings_ranking_rules.rs b/meilisearch-http/_tests/settings_ranking_rules.rs new file mode 100644 index 000000000..ac9a1e00c --- /dev/null +++ b/meilisearch-http/_tests/settings_ranking_rules.rs @@ -0,0 +1,182 @@ +use assert_json_diff::assert_json_eq; +use serde_json::json; + +mod common; + +#[actix_rt::test] +async fn write_all_and_delete() { + let mut server = common::Server::test_server().await; + + // 2 - Send the settings + + let body = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ]); + + server.update_ranking_rules(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (response, _status_code) = server.get_ranking_rules().await; + + assert_json_eq!(body, response, ordered: false); + + // 4 - Delete all settings + + server.delete_ranking_rules().await; + + // 5 - Get all settings and check if they are empty + + let (response, _status_code) = server.get_ranking_rules().await; + + let expected = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ]); + + assert_json_eq!(expected, response, ordered: false); +} + +#[actix_rt::test] +async fn write_all_and_update() { + let mut server = common::Server::test_server().await; + + // 2 - Send the settings + + let body = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + "desc(age)", + ]); + + server.update_ranking_rules(body.clone()).await; + + // 3 - Get all settings and compare to the previous one + + let (response, _status_code) = server.get_ranking_rules().await; + + assert_json_eq!(body, response, ordered: false); + + // 4 - Update all settings + + let body = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + ]); + + server.update_ranking_rules(body).await; + + // 5 - Get all settings and check if the content is the same of (4) + + let (response, _status_code) = server.get_ranking_rules().await; + + let expected = json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "desc(registered)", + ]); + + assert_json_eq!(expected, response, ordered: false); +} + +#[actix_rt::test] +async fn send_undefined_rule() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + let body = json!(["typos",]); + + let (_response, status_code) = server.update_ranking_rules_sync(body).await; + assert_eq!(status_code, 400); +} + +#[actix_rt::test] +async fn send_malformed_custom_rule() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + let body = json!(["dsc(truc)",]); + + let (_response, status_code) = server.update_ranking_rules_sync(body).await; + assert_eq!(status_code, 400); +} + +// Test issue https://github.com/meilisearch/MeiliSearch/issues/521 +#[actix_rt::test] +async fn write_custom_ranking_and_index_documents() { + let mut server = common::Server::with_uid("test"); + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + server.create_index(body).await; + + // 1 - Add ranking rules with one custom ranking on a string + + let body = json!(["asc(name)", "typo"]); + + server.update_ranking_rules(body).await; + + // 2 - Add documents + + let body = json!([ + { + "id": 1, + "name": "Cherry Orr", + "color": "green" + }, + { + "id": 2, + "name": "Lucas Hess", + "color": "yellow" + } + ]); + + server.add_or_replace_multiple_documents(body).await; + + // 3 - Get the first document and compare + + let expected = json!({ + "id": 1, + "name": "Cherry Orr", + "color": "green" + }); + + let (response, status_code) = server.get_document(1).await; + assert_eq!(status_code, 200); + + assert_json_eq!(response, expected, ordered: false); +} diff --git a/meilisearch-http/_tests/settings_stop_words.rs b/meilisearch-http/_tests/settings_stop_words.rs new file mode 100644 index 000000000..3ff2e8bb7 --- /dev/null +++ b/meilisearch-http/_tests/settings_stop_words.rs @@ -0,0 +1,61 @@ +use assert_json_diff::assert_json_eq; +use serde_json::json; + +mod common; + +#[actix_rt::test] +async fn update_stop_words() { + let mut server = common::Server::test_server().await; + + // 1 - Get stop words + + let (response, _status_code) = server.get_stop_words().await; + assert_eq!(response.as_array().unwrap().is_empty(), true); + + // 2 - Update stop words + + let body = json!(["ut", "ea"]); + server.update_stop_words(body.clone()).await; + + // 3 - Get all stop words and compare to the previous one + + let (response, _status_code) = server.get_stop_words().await; + assert_json_eq!(body, response, ordered: false); + + // 4 - Delete all stop words + + server.delete_stop_words().await; + + // 5 - Get all stop words and check if they are empty + + let (response, _status_code) = server.get_stop_words().await; + assert_eq!(response.as_array().unwrap().is_empty(), true); +} + +#[actix_rt::test] +async fn add_documents_and_stop_words() { + let mut server = common::Server::test_server().await; + + // 2 - Update stop words + + let body = json!(["ad", "in"]); + server.update_stop_words(body.clone()).await; + + // 3 - Search for a document with stop words + + let (response, _status_code) = server.search_get("q=in%20exercitation").await; + assert!(!response["hits"].as_array().unwrap().is_empty()); + + // 4 - Search for documents with *only* stop words + + let (response, _status_code) = server.search_get("q=ad%20in").await; + assert!(response["hits"].as_array().unwrap().is_empty()); + + // 5 - Delete all stop words + + // server.delete_stop_words(); + + // // 6 - Search for a document with one stop word + + // assert!(!response["hits"].as_array().unwrap().is_empty()); +} diff --git a/meilisearch-http/_tests/url_normalizer.rs b/meilisearch-http/_tests/url_normalizer.rs new file mode 100644 index 000000000..c2c9187ee --- /dev/null +++ b/meilisearch-http/_tests/url_normalizer.rs @@ -0,0 +1,18 @@ +mod common; + +#[actix_rt::test] +async fn url_normalizer() { + let mut server = common::Server::with_uid("movies"); + + let (_response, status_code) = server.get_request("/version").await; + assert_eq!(status_code, 200); + + let (_response, status_code) = server.get_request("//version").await; + assert_eq!(status_code, 200); + + let (_response, status_code) = server.get_request("/version/").await; + assert_eq!(status_code, 200); + + let (_response, status_code) = server.get_request("//version/").await; + assert_eq!(status_code, 200); +} diff --git a/build.rs b/meilisearch-http/build.rs similarity index 100% rename from build.rs rename to meilisearch-http/build.rs diff --git a/meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/data.mdb b/meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/data.mdb new file mode 100644 index 0000000000000000000000000000000000000000..e1c9db873b20be58dd26e5c90038cb00586ddddb GIT binary patch literal 32768 zcmeI)O-{l<7{>9}`h_u27c2}KZ_vaAD{n&tlNz7~@N4G@IEyDREH7gZ!ASymUp; ztEjm>>bIHq-_A?5=bID=AbMa(RSkDNtw@n?-$v+JbSj4m2+FS_U@H*qH(+|C%%U3qn6mtL@ooAl#B zoOGu?Jr`@&on01i>Ja&z*sx9`?oN9f*Zl#3)ofU)heL9IZ$bqD1Q0*~0R#|0009IL zK;Ul#{Fnmy|M$RSBQdk({~P{$0;}46_#EDczKIF~2q1s}0tg_000IagfB*tZBjCmq z{>%Kodl_HJ^xizJ-`8{V+Ezd1_iESs^>n<-(Q1%GJIUt%S^d8>KWop#rd|&<4u72a=1fQU%;rajo literal 0 HcmV?d00001 diff --git a/meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/lock.mdb b/meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/lock.mdb new file mode 100644 index 0000000000000000000000000000000000000000..7bae5680e543d283e07ca391af515f86818e93f1 GIT binary patch literal 8192 zcmeIup$z~a3`Idutj|P*NmzjuFbL9sLtb5f^307sTTVH}t8v(Wx$I~C>Mogy009C7 k2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5cnjp0Eb)zb^rhX literal 0 HcmV?d00001 diff --git a/meilisearch-http/data.ms/indexes/index-eb9d0392-4386-4fa1-b68c-dac0d93bb16e/data.mdb b/meilisearch-http/data.ms/indexes/index-eb9d0392-4386-4fa1-b68c-dac0d93bb16e/data.mdb new file mode 100644 index 0000000000000000000000000000000000000000..92d43bcc390866e63d79078d2ad9d880b50b004c GIT binary patch literal 45056 zcmeI*!B5jr9Ki9ni8vr@7sJ5_@z9GGnlNKD>AxX&Qdt)ZY+cqO5JFz(~X=l(yf1G4|Zh6(}*AbVFqU{F!*W}_^vI3lHJ6;Q<#@~cX&J9(jnW3o zj+gFOa+zC!rCF&t?I?Qg_nnj5#l@V?|9jSBo4fxD5-JEFfB*sr zAbCJis9BWPV@T$@==kpVQSKPUruu{&&U+U^4;; zAbl)~fo?ObDs;fa*rHY}+$ zxyzG!x9-2*H6P`{m80AbM?X$;IjIB)5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U KAV7e?AAuXq{|@{B literal 0 HcmV?d00001 diff --git a/meilisearch-http/data.ms/updates/data.mdb b/meilisearch-http/data.ms/updates/data.mdb new file mode 100644 index 0000000000000000000000000000000000000000..dea8bd465ebff560d5aa6cf37a8b63b65e23eb00 GIT binary patch literal 65536 zcmeI*y>8o96u|LY^~0|1ibhcsL5G0gYy~9sVM$X0bZMueXqOa4K#A9}8i`UQ%821= z%iu9j5@hU96n%mWojUau3iJh9ejUWj~4k=l7rMj4ucvfB*srAb*~Bsv~6!V9(ZB2?dwU>b~Nq{yz$|qC;HH82h)Ckwqxnw=~Pd({~&JX+OAWv-HN;W zo!x3XyY0Gj&uKNAZmZhdeHiuF3w-_T`)My8P25(rGli72oUH zeyiH)I1L@`G6}tLsQquo!>*o8dcos2j1&!3Z#l_GdORMEqfvUnf!FW(W=zmF2i|DM z)Xzq`8+B=V=0rzFA9S_pg=W7OXwxyf!~S$&CBTL24@Xvf%%-zrTHRp~Mu^e32x!>x z#+EtzaYOR&as2;A;U}5S%}J+CmOO$(dyPFLfp#Pm%}aPtLNROuKQ_ytoEnUf3@${uliibRPLq6wqorv#6k7{wV%w zb@q~(@{M|{TCZ2Dc05!5I@KGzgn9gLS#K~KspT|Xw^n;-%{B`4xc>h^`GENU%CCL~ zKmY**5I_I{1Q0*~0R(Q3zfL6SEcxK7pCZPe^I{-1rE zUvwq)x!O1Ov-*Ge&#WMT00IagfB*srAbf7g)u|5-B@2q1s} z0tg_000IagfB*sr+ya4{^Zfs^_5bC5LLC1~{{IE<|GNd(Hm8UH0tg_000IagfB*sr zTxWrsTmN73@si({{QF{lznSo_D~Zb`zaNF<$w%_{lDC)blK+=}WjtvozsvaY-aqM& z_y1+UED%5d0R#|0009ILKmY**u7SYKjsH*Lz485j@?d~c#ag4hyA!!DAkP&L|DXI1v5o)&2q1s}0tg_000Iag@NNYz5dTX;J&&5b&Tt%R z|3uV${<+OZ?q!v~bl?KLtV#Ty4lhyu*`d3n`tvhcCjOT?0W$v}j{iSXFN|M|c=kUZ z1Q0*~0R#|0009ILKmY**5V%SKnYVvl-9C9vV6hIMjX&t|36ykJ+c4* literal 0 HcmV?d00001 diff --git a/meilisearch-http/data.ms/updates/lock.mdb b/meilisearch-http/data.ms/updates/lock.mdb new file mode 100644 index 0000000000000000000000000000000000000000..b5c5d308cf28870d5d05eb7ef611b55192e6668a GIT binary patch literal 8192 zcmeIuu?>JA5CA}^UCL=(!xb#;+{8GErL7}q6i7&ji38jh|IhPK+`gS_3^6>B^4o?f zl_qz2Qt#IN*SqGUJh*a{`~FzwVJs(=009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs M0RjXF5cnf-1Ev!Xwg3PC literal 0 HcmV?d00001 diff --git a/public/bulma.min.css b/meilisearch-http/public/bulma.min.css similarity index 100% rename from public/bulma.min.css rename to meilisearch-http/public/bulma.min.css diff --git a/public/interface.html b/meilisearch-http/public/interface.html similarity index 100% rename from public/interface.html rename to meilisearch-http/public/interface.html diff --git a/src/analytics.rs b/meilisearch-http/src/analytics.rs similarity index 100% rename from src/analytics.rs rename to meilisearch-http/src/analytics.rs diff --git a/src/data/mod.rs b/meilisearch-http/src/data/mod.rs similarity index 100% rename from src/data/mod.rs rename to meilisearch-http/src/data/mod.rs diff --git a/src/data/search.rs b/meilisearch-http/src/data/search.rs similarity index 100% rename from src/data/search.rs rename to meilisearch-http/src/data/search.rs diff --git a/src/data/updates.rs b/meilisearch-http/src/data/updates.rs similarity index 100% rename from src/data/updates.rs rename to meilisearch-http/src/data/updates.rs diff --git a/src/dump.rs b/meilisearch-http/src/dump.rs similarity index 100% rename from src/dump.rs rename to meilisearch-http/src/dump.rs diff --git a/src/error.rs b/meilisearch-http/src/error.rs similarity index 100% rename from src/error.rs rename to meilisearch-http/src/error.rs diff --git a/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs similarity index 100% rename from src/helpers/authentication.rs rename to meilisearch-http/src/helpers/authentication.rs diff --git a/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs similarity index 100% rename from src/helpers/compression.rs rename to meilisearch-http/src/helpers/compression.rs diff --git a/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs similarity index 100% rename from src/helpers/mod.rs rename to meilisearch-http/src/helpers/mod.rs diff --git a/src/helpers/normalize_path.rs b/meilisearch-http/src/helpers/normalize_path.rs similarity index 100% rename from src/helpers/normalize_path.rs rename to meilisearch-http/src/helpers/normalize_path.rs diff --git a/meilisearch-http/src/index_controller/actor_index_controller/update_handler.rs b/meilisearch-http/src/index_controller/actor_index_controller/update_handler.rs new file mode 100644 index 000000000..d9ac2f866 --- /dev/null +++ b/meilisearch-http/src/index_controller/actor_index_controller/update_handler.rs @@ -0,0 +1,260 @@ +use std::collections::HashMap; +use std::io; +use std::fs::File; + +use anyhow::Result; +use flate2::read::GzDecoder; +use grenad::CompressionType; +use log::info; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::Index; +use rayon::ThreadPool; + +use crate::index_controller::updates::{Failed, Processed, Processing}; +use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; +use crate::option::IndexerOpts; + +pub struct UpdateHandler { + max_nb_chunks: Option, + chunk_compression_level: Option, + thread_pool: ThreadPool, + log_frequency: usize, + max_memory: usize, + linked_hash_map_size: usize, + chunk_compression_type: CompressionType, + chunk_fusing_shrink_size: u64, +} + +impl UpdateHandler { + pub fn new( + opt: &IndexerOpts, + ) -> anyhow::Result { + let thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(opt.indexing_jobs.unwrap_or(0)) + .build()?; + Ok(Self { + max_nb_chunks: opt.max_nb_chunks, + chunk_compression_level: opt.chunk_compression_level, + thread_pool, + log_frequency: opt.log_every_n, + max_memory: opt.max_memory.get_bytes() as usize, + linked_hash_map_size: opt.linked_hash_map_size, + chunk_compression_type: opt.chunk_compression_type, + chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), + }) + } + + fn update_buidler(&self, update_id: u64) -> UpdateBuilder { + // We prepare the update by using the update builder. + let mut update_builder = UpdateBuilder::new(update_id); + if let Some(max_nb_chunks) = self.max_nb_chunks { + update_builder.max_nb_chunks(max_nb_chunks); + } + if let Some(chunk_compression_level) = self.chunk_compression_level { + update_builder.chunk_compression_level(chunk_compression_level); + } + update_builder.thread_pool(&self.thread_pool); + update_builder.log_every_n(self.log_frequency); + update_builder.max_memory(self.max_memory); + update_builder.linked_hash_map_size(self.linked_hash_map_size); + update_builder.chunk_compression_type(self.chunk_compression_type); + update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); + update_builder + } + + fn update_documents( + &self, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: File, + update_builder: UpdateBuilder, + primary_key: Option<&str>, + index: &Index, + ) -> anyhow::Result { + info!("performing document addition"); + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + + // Set the primary key if not set already, ignore if already set. + match (index.primary_key(&wtxn)?, primary_key) { + (None, Some(ref primary_key)) => { + index.put_primary_key(&mut wtxn, primary_key)?; + } + _ => (), + } + + let mut builder = update_builder.index_documents(&mut wtxn, index); + builder.update_format(format); + builder.index_documents_method(method); + + let gzipped = false; + let reader = if gzipped { + Box::new(GzDecoder::new(content)) + } else { + Box::new(content) as Box + }; + + let result = builder.execute(reader, |indexing_step, update_id| { + info!("update {}: {:?}", update_id, indexing_step) + }); + + info!("document addition done: {:?}", result); + + match result { + Ok(addition_result) => wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn clear_documents(&self, update_builder: UpdateBuilder, index: &Index) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, index); + + match builder.execute() { + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn update_settings( + &self, + settings: &Settings, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let mut builder = update_builder.settings(&mut wtxn, index); + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names.clone()), + None => builder.reset_searchable_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names.clone()), + None => builder.reset_displayed_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref facet_types) = settings.faceted_attributes { + let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + builder.set_faceted_fields(facet_types); + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref criteria) = settings.criteria { + match criteria { + Some(criteria) => builder.set_criteria(criteria.clone()), + None => builder.reset_criteria(), + } + } + + let result = builder + .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn update_facets( + &self, + levels: &Facets, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let mut builder = update_builder.facets(&mut wtxn, index); + if let Some(value) = levels.level_group_size { + builder.level_group_size(value); + } + if let Some(value) = levels.min_level_size { + builder.min_level_size(value); + } + match builder.execute() { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn delete_documents( + &self, + document_ids: File, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + let ids: Vec = serde_json::from_reader(document_ids)?; + let mut txn = index.write_txn()?; + let mut builder = update_builder.delete_documents(&mut txn, index)?; + + // We ignore unexisting document ids + ids.iter().for_each(|id| { builder.delete_external_id(id); }); + + match builder.execute() { + Ok(deleted) => txn + .commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + pub fn handle_update( + &self, + meta: Processing, + content: File, + index: &Index, + ) -> Result, Failed> { + use UpdateMeta::*; + + let update_id = meta.id(); + + let update_builder = self.update_buidler(update_id); + + let result = match meta.meta() { + DocumentsAddition { + method, + format, + primary_key, + } => self.update_documents( + *format, + *method, + content, + update_builder, + primary_key.as_deref(), + index, + ), + ClearDocuments => self.clear_documents(update_builder, index), + DeleteDocuments => self.delete_documents(content, update_builder, index), + Settings(settings) => self.update_settings(settings, update_builder, index), + Facets(levels) => self.update_facets(levels, update_builder, index), + }; + + match result { + Ok(result) => Ok(meta.process(result)), + Err(e) => Err(meta.fail(e.to_string())), + } + } +} diff --git a/meilisearch-http/src/index_controller/actor_index_controller/update_store.rs b/meilisearch-http/src/index_controller/actor_index_controller/update_store.rs new file mode 100644 index 000000000..371ac7bd9 --- /dev/null +++ b/meilisearch-http/src/index_controller/actor_index_controller/update_store.rs @@ -0,0 +1,423 @@ +use std::path::Path; +use std::sync::{Arc, RwLock}; +use std::io::{Cursor, SeekFrom, Seek}; + +use crossbeam_channel::Sender; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::{EnvOpenOptions, Env, Database}; +use serde::{Serialize, Deserialize}; +use std::fs::File; +use uuid::Uuid; + +use crate::index_controller::updates::*; + +type BEU64 = heed::zerocopy::U64; + +#[derive(Clone)] +pub struct UpdateStore { + env: Env, + pending_meta: Database, SerdeJson>>, + pending: Database, ByteSlice>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, + notification_sender: Sender<()>, +} + +pub trait HandleUpdate { + fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed>; +} + +impl HandleUpdate for F +where F: FnMut(Processing, File) -> Result, Failed> +{ + fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed> { + self(meta, content) + } +} + +impl UpdateStore +where + M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, + N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, + E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, +{ + pub fn open( + mut options: EnvOpenOptions, + path: P, + mut update_handler: U, + ) -> heed::Result> + where + P: AsRef, + U: HandleUpdate + Send + 'static, + { + options.max_dbs(5); + + let env = options.open(path)?; + let pending_meta = env.create_database(Some("pending-meta"))?; + let pending = env.create_database(Some("pending"))?; + let processed_meta = env.create_database(Some("processed-meta"))?; + let aborted_meta = env.create_database(Some("aborted-meta"))?; + let failed_meta = env.create_database(Some("failed-meta"))?; + let processing = Arc::new(RwLock::new(None)); + + let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + // Send a first notification to trigger the process. + let _ = notification_sender.send(()); + + let update_store = Arc::new(UpdateStore { + env, + pending, + pending_meta, + processed_meta, + aborted_meta, + notification_sender, + failed_meta, + processing, + }); + + // We need a weak reference so we can take ownership on the arc later when we + // want to close the index. + let update_store_weak = Arc::downgrade(&update_store); + std::thread::spawn(move || { + // Block and wait for something to process. + 'outer: for _ in notification_receiver { + loop { + match update_store_weak.upgrade() { + Some(update_store) => { + match update_store.process_pending_update(&mut update_handler) { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => eprintln!("error while processing update: {}", e), + } + } + // the ownership on the arc has been taken, we need to exit. + None => break 'outer, + } + } + } + }); + + Ok(update_store) + } + + pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { + self.env.prepare_for_closing() + } + + /// Returns the new biggest id to use to store the new update. + fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { + let last_pending = self.pending_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_processed = self.processed_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_aborted = self.aborted_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_update_id = [last_pending, last_processed, last_aborted] + .iter() + .copied() + .flatten() + .max(); + + match last_update_id { + Some(last_id) => Ok(last_id + 1), + None => Ok(0), + } + } + + /// Registers the update content in the pending store and the meta + /// into the pending-meta store. Returns the new unique update id. + pub fn register_update( + &self, + meta: M, + content: &[u8], + index_uuid: Uuid, + ) -> heed::Result> { + let mut wtxn = self.env.write_txn()?; + + // We ask the update store to give us a new update id, this is safe, + // no other update can have the same id because we use a write txn before + // asking for the id and registering it so other update registering + // will be forced to wait for a new write txn. + let update_id = self.new_update_id(&wtxn)?; + let update_key = BEU64::new(update_id); + + let meta = Pending::new(meta, update_id, index_uuid); + self.pending_meta.put(&mut wtxn, &update_key, &meta)?; + self.pending.put(&mut wtxn, &update_key, content)?; + + wtxn.commit()?; + + if let Err(e) = self.notification_sender.try_send(()) { + assert!(!e.is_disconnected(), "update notification channel is disconnected"); + } + Ok(meta) + } + /// Executes the user provided function on the next pending update (the one with the lowest id). + /// This is asynchronous as it let the user process the update with a read-only txn and + /// only writing the result meta to the processed-meta store *after* it has been processed. + fn process_pending_update(&self, handler: &mut U) -> heed::Result> + where + U: HandleUpdate + Send + 'static, + { + // Create a read transaction to be able to retrieve the pending update in order. + let rtxn = self.env.read_txn()?; + let first_meta = self.pending_meta.first(&rtxn)?; + + // If there is a pending update we process and only keep + // a reader while processing it, not a writer. + match first_meta { + Some((first_id, pending)) => { + let first_content = self.pending + .get(&rtxn, &first_id)? + .expect("associated update content"); + + // we change the state of the update from pending to processing before we pass it + // to the update handler. Processing store is non persistent to be able recover + // from a failure + let processing = pending.processing(); + self.processing + .write() + .unwrap() + .replace(processing.clone()); + let mut cursor = Cursor::new(first_content); + let mut file = tempfile::tempfile()?; + std::io::copy(&mut cursor, &mut file)?; + file.seek(SeekFrom::Start(0))?; + // Process the pending update using the provided user function. + let result = handler.handle_update(processing, file); + drop(rtxn); + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.processing + .write() + .unwrap() + .take(); + self.pending_meta.delete(&mut wtxn, &first_id)?; + self.pending.delete(&mut wtxn, &first_id)?; + match result { + Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, + Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, + } + wtxn.commit()?; + + Ok(Some(())) + }, + None => Ok(None) + } + } + + /// Execute the user defined function with the meta-store iterators, the first + /// iterator is the *processed* meta one, the second the *aborted* meta one + /// and, the last is the *pending* meta one. + pub fn iter_metas(&self, mut f: F) -> heed::Result + where + F: for<'a> FnMut( + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + ) -> heed::Result, + { + let rtxn = self.env.read_txn()?; + + // We get the pending, processed and aborted meta iterators. + let processed_iter = self.processed_meta.iter(&rtxn)?; + let aborted_iter = self.aborted_meta.iter(&rtxn)?; + let pending_iter = self.pending_meta.iter(&rtxn)?; + let processing = self.processing.read().unwrap().clone(); + let failed_iter = self.failed_meta.iter(&rtxn)?; + + // We execute the user defined function with both iterators. + (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + } + + /// Returns the update associated meta or `None` if the update doesn't exist. + pub fn meta(&self, update_id: u64) -> heed::Result>> { + let rtxn = self.env.read_txn()?; + let key = BEU64::new(update_id); + + if let Some(ref meta) = *self.processing.read().unwrap() { + if meta.id() == update_id { + return Ok(Some(UpdateStatus::Processing(meta.clone()))); + } + } + + if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Pending(meta))); + } + + if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Processed(meta))); + } + + if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Aborted(meta))); + } + + if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Failed(meta))); + } + + Ok(None) + } + + /// Aborts an update, an aborted update content is deleted and + /// the meta of it is moved into the aborted updates database. + /// + /// Trying to abort an update that is currently being processed, an update + /// that as already been processed or which doesn't actually exist, will + /// return `None`. + #[allow(dead_code)] + pub fn abort_update(&self, update_id: u64) -> heed::Result>> { + let mut wtxn = self.env.write_txn()?; + let key = BEU64::new(update_id); + + // We cannot abort an update that is currently being processed. + if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { + return Ok(None); + } + + let pending = match self.pending_meta.get(&wtxn, &key)? { + Some(meta) => meta, + None => return Ok(None), + }; + + let aborted = pending.abort(); + + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + + wtxn.commit()?; + + Ok(Some(aborted)) + } + + /// Aborts all the pending updates, and not the one being currently processed. + /// Returns the update metas and ids that were successfully aborted. + #[allow(dead_code)] + pub fn abort_pendings(&self) -> heed::Result)>> { + let mut wtxn = self.env.write_txn()?; + let mut aborted_updates = Vec::new(); + + // We skip the first pending update as it is currently being processed. + for result in self.pending_meta.iter(&wtxn)?.skip(1) { + let (key, pending) = result?; + let id = key.get(); + aborted_updates.push((id, pending.abort())); + } + + for (id, aborted) in &aborted_updates { + let key = BEU64::new(*id); + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + } + + wtxn.commit()?; + + Ok(aborted_updates) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::{Duration, Instant}; + + impl HandleUpdate for F + where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { + fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { + self(meta, content) + } + } + + #[test] + fn simple() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let meta = String::from("kiki"); + let update = update_store.register_update(meta, &[]).unwrap(); + thread::sleep(Duration::from_millis(100)); + let meta = update_store.meta(update.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + } + + #[test] + #[ignore] + fn long_running_update() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let before_register = Instant::now(); + + let meta = String::from("kiki"); + let update_kiki = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("coco"); + let update_coco = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("cucu"); + let update_cucu = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + thread::sleep(Duration::from_millis(400 * 3 + 100)); + + let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "coco processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "cucu processed"); + } else { + panic!() + } + } +} diff --git a/src/index_controller/local_index_controller/index_store.rs b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs similarity index 100% rename from src/index_controller/local_index_controller/index_store.rs rename to meilisearch-http/src/index_controller/local_index_controller/index_store.rs diff --git a/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs similarity index 100% rename from src/index_controller/local_index_controller/mod.rs rename to meilisearch-http/src/index_controller/local_index_controller/mod.rs diff --git a/src/index_controller/local_index_controller/update_handler.rs b/meilisearch-http/src/index_controller/local_index_controller/update_handler.rs similarity index 100% rename from src/index_controller/local_index_controller/update_handler.rs rename to meilisearch-http/src/index_controller/local_index_controller/update_handler.rs diff --git a/src/index_controller/local_index_controller/update_store.rs b/meilisearch-http/src/index_controller/local_index_controller/update_store.rs similarity index 100% rename from src/index_controller/local_index_controller/update_store.rs rename to meilisearch-http/src/index_controller/local_index_controller/update_store.rs diff --git a/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs similarity index 100% rename from src/index_controller/mod.rs rename to meilisearch-http/src/index_controller/mod.rs diff --git a/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs similarity index 100% rename from src/index_controller/updates.rs rename to meilisearch-http/src/index_controller/updates.rs diff --git a/src/lib.rs b/meilisearch-http/src/lib.rs similarity index 100% rename from src/lib.rs rename to meilisearch-http/src/lib.rs diff --git a/src/main.rs b/meilisearch-http/src/main.rs similarity index 100% rename from src/main.rs rename to meilisearch-http/src/main.rs diff --git a/src/option.rs b/meilisearch-http/src/option.rs similarity index 100% rename from src/option.rs rename to meilisearch-http/src/option.rs diff --git a/src/routes/document.rs b/meilisearch-http/src/routes/document.rs similarity index 100% rename from src/routes/document.rs rename to meilisearch-http/src/routes/document.rs diff --git a/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs similarity index 100% rename from src/routes/dump.rs rename to meilisearch-http/src/routes/dump.rs diff --git a/src/routes/health.rs b/meilisearch-http/src/routes/health.rs similarity index 100% rename from src/routes/health.rs rename to meilisearch-http/src/routes/health.rs diff --git a/src/routes/index.rs b/meilisearch-http/src/routes/index.rs similarity index 100% rename from src/routes/index.rs rename to meilisearch-http/src/routes/index.rs diff --git a/src/routes/key.rs b/meilisearch-http/src/routes/key.rs similarity index 100% rename from src/routes/key.rs rename to meilisearch-http/src/routes/key.rs diff --git a/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs similarity index 100% rename from src/routes/mod.rs rename to meilisearch-http/src/routes/mod.rs diff --git a/src/routes/search.rs b/meilisearch-http/src/routes/search.rs similarity index 100% rename from src/routes/search.rs rename to meilisearch-http/src/routes/search.rs diff --git a/src/routes/settings/attributes_for_faceting.rs b/meilisearch-http/src/routes/settings/attributes_for_faceting.rs similarity index 100% rename from src/routes/settings/attributes_for_faceting.rs rename to meilisearch-http/src/routes/settings/attributes_for_faceting.rs diff --git a/src/routes/settings/displayed_attributes.rs b/meilisearch-http/src/routes/settings/displayed_attributes.rs similarity index 100% rename from src/routes/settings/displayed_attributes.rs rename to meilisearch-http/src/routes/settings/displayed_attributes.rs diff --git a/src/routes/settings/distinct_attributes.rs b/meilisearch-http/src/routes/settings/distinct_attributes.rs similarity index 100% rename from src/routes/settings/distinct_attributes.rs rename to meilisearch-http/src/routes/settings/distinct_attributes.rs diff --git a/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs similarity index 100% rename from src/routes/settings/mod.rs rename to meilisearch-http/src/routes/settings/mod.rs diff --git a/src/routes/settings/ranking_rules.rs b/meilisearch-http/src/routes/settings/ranking_rules.rs similarity index 100% rename from src/routes/settings/ranking_rules.rs rename to meilisearch-http/src/routes/settings/ranking_rules.rs diff --git a/src/routes/settings/searchable_attributes.rs b/meilisearch-http/src/routes/settings/searchable_attributes.rs similarity index 100% rename from src/routes/settings/searchable_attributes.rs rename to meilisearch-http/src/routes/settings/searchable_attributes.rs diff --git a/src/routes/settings/stop_words.rs b/meilisearch-http/src/routes/settings/stop_words.rs similarity index 100% rename from src/routes/settings/stop_words.rs rename to meilisearch-http/src/routes/settings/stop_words.rs diff --git a/src/routes/settings/synonyms.rs b/meilisearch-http/src/routes/settings/synonyms.rs similarity index 100% rename from src/routes/settings/synonyms.rs rename to meilisearch-http/src/routes/settings/synonyms.rs diff --git a/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs similarity index 100% rename from src/routes/stats.rs rename to meilisearch-http/src/routes/stats.rs diff --git a/src/routes/stop_words.rs b/meilisearch-http/src/routes/stop_words.rs similarity index 100% rename from src/routes/stop_words.rs rename to meilisearch-http/src/routes/stop_words.rs diff --git a/src/routes/synonym.rs b/meilisearch-http/src/routes/synonym.rs similarity index 100% rename from src/routes/synonym.rs rename to meilisearch-http/src/routes/synonym.rs diff --git a/src/snapshot.rs b/meilisearch-http/src/snapshot.rs similarity index 100% rename from src/snapshot.rs rename to meilisearch-http/src/snapshot.rs diff --git a/meilisearch-http/tests/assets/dumps/v1/metadata.json b/meilisearch-http/tests/assets/dumps/v1/metadata.json new file mode 100644 index 000000000..6fe302324 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/metadata.json @@ -0,0 +1,12 @@ +{ + "indices": [{ + "uid": "test", + "primaryKey": "id" + }, { + "uid": "test2", + "primaryKey": "test2_id" + } + ], + "dbVersion": "0.13.0", + "dumpVersion": "1" +} diff --git a/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl new file mode 100644 index 000000000..7af80f342 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl @@ -0,0 +1,77 @@ +{"id":0,"isActive":false,"balance":"$2,668.55","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Lucas Hess","gender":"male","email":"lucashess@chorizon.com","phone":"+1 (998) 478-2597","address":"412 Losee Terrace, Blairstown, Georgia, 2825","about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n","registered":"2016-06-21T09:30:25 -02:00","latitude":-44.174957,"longitude":-145.725388,"tags":["bug","bug"]} +{"id":1,"isActive":true,"balance":"$1,706.13","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Cherry Orr","gender":"female","email":"cherryorr@chorizon.com","phone":"+1 (995) 479-3174","address":"442 Beverly Road, Ventress, New Mexico, 3361","about":"Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n","registered":"2020-03-18T11:12:21 -01:00","latitude":-24.356932,"longitude":27.184808,"tags":["new issue","bug"]} +{"id":2,"isActive":true,"balance":"$2,467.47","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Patricia Goff","gender":"female","email":"patriciagoff@chorizon.com","phone":"+1 (864) 463-2277","address":"866 Hornell Loop, Cresaptown, Ohio, 1700","about":"Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n","registered":"2014-10-28T12:59:30 -01:00","latitude":-64.008555,"longitude":11.867098,"tags":["good first issue"]} +{"id":3,"isActive":true,"balance":"$3,344.40","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Adeline Flynn","gender":"female","email":"adelineflynn@chorizon.com","phone":"+1 (994) 600-2840","address":"428 Paerdegat Avenue, Hollymead, Pennsylvania, 948","about":"Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n","registered":"2014-03-27T06:24:45 -01:00","latitude":-74.485173,"longitude":-11.059859,"tags":["bug","good first issue","wontfix","new issue"]} +{"id":4,"isActive":false,"balance":"$2,575.78","picture":"http://placehold.it/32x32","age":39,"color":"Green","name":"Mariana Pacheco","gender":"female","email":"marianapacheco@chorizon.com","phone":"+1 (820) 414-2223","address":"664 Rapelye Street, Faywood, California, 7320","about":"Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n","registered":"2015-09-02T03:23:35 -02:00","latitude":75.763501,"longitude":-78.777124,"tags":["new issue"]} +{"id":5,"isActive":true,"balance":"$3,793.09","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Warren Watson","gender":"male","email":"warrenwatson@chorizon.com","phone":"+1 (807) 583-2427","address":"671 Prince Street, Faxon, Connecticut, 4275","about":"Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n","registered":"2017-06-04T06:02:17 -02:00","latitude":29.979223,"longitude":25.358943,"tags":["wontfix","wontfix","wontfix"]} +{"id":6,"isActive":true,"balance":"$2,919.70","picture":"http://placehold.it/32x32","age":20,"color":"blue","name":"Shelia Berry","gender":"female","email":"sheliaberry@chorizon.com","phone":"+1 (853) 511-2651","address":"437 Forrest Street, Coventry, Illinois, 2056","about":"Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n","registered":"2018-07-11T02:45:01 -02:00","latitude":54.815991,"longitude":-118.690609,"tags":["good first issue","bug","wontfix","new issue"]} +{"id":7,"isActive":true,"balance":"$1,349.50","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Chrystal Boyd","gender":"female","email":"chrystalboyd@chorizon.com","phone":"+1 (936) 563-2802","address":"670 Croton Loop, Sussex, Florida, 4692","about":"Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n","registered":"2016-11-01T07:36:04 -01:00","latitude":-24.711933,"longitude":147.246705,"tags":[]} +{"id":8,"isActive":false,"balance":"$3,999.56","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Martin Porter","gender":"male","email":"martinporter@chorizon.com","phone":"+1 (895) 580-2304","address":"577 Regent Place, Aguila, Guam, 6554","about":"Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n","registered":"2014-09-20T02:08:30 -02:00","latitude":-88.344273,"longitude":37.964466,"tags":[]} +{"id":9,"isActive":true,"balance":"$3,729.71","picture":"http://placehold.it/32x32","age":26,"color":"blue","name":"Kelli Mendez","gender":"female","email":"kellimendez@chorizon.com","phone":"+1 (936) 401-2236","address":"242 Caton Place, Grazierville, Alabama, 3968","about":"Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n","registered":"2018-05-04T10:35:30 -02:00","latitude":49.37551,"longitude":41.872323,"tags":["new issue","new issue"]} +{"id":10,"isActive":false,"balance":"$1,127.47","picture":"http://placehold.it/32x32","age":27,"color":"blue","name":"Maddox Johns","gender":"male","email":"maddoxjohns@chorizon.com","phone":"+1 (892) 470-2357","address":"756 Beard Street, Avalon, Louisiana, 114","about":"Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n","registered":"2016-04-22T06:41:25 -02:00","latitude":66.640229,"longitude":-17.222666,"tags":["new issue","good first issue","good first issue","new issue"]} +{"id":11,"isActive":true,"balance":"$1,351.43","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Evans Wagner","gender":"male","email":"evanswagner@chorizon.com","phone":"+1 (889) 496-2332","address":"118 Monaco Place, Lutsen, Delaware, 6209","about":"Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n","registered":"2016-10-27T01:26:31 -02:00","latitude":-77.673222,"longitude":-142.657214,"tags":["good first issue","good first issue"]} +{"id":12,"isActive":false,"balance":"$3,394.96","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Aida Kirby","gender":"female","email":"aidakirby@chorizon.com","phone":"+1 (942) 532-2325","address":"797 Engert Avenue, Wilsonia, Idaho, 6532","about":"Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n","registered":"2018-06-18T04:39:57 -02:00","latitude":-58.062041,"longitude":34.999254,"tags":["new issue","wontfix","bug","new issue"]} +{"id":13,"isActive":true,"balance":"$2,812.62","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Nelda Burris","gender":"female","email":"neldaburris@chorizon.com","phone":"+1 (813) 600-2576","address":"160 Opal Court, Fowlerville, Tennessee, 2170","about":"Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n","registered":"2015-08-15T12:39:53 -02:00","latitude":66.6871,"longitude":179.549488,"tags":["wontfix"]} +{"id":14,"isActive":true,"balance":"$1,718.33","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Jennifer Hart","gender":"female","email":"jenniferhart@chorizon.com","phone":"+1 (850) 537-2513","address":"124 Veranda Place, Nash, Utah, 985","about":"Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n","registered":"2016-09-04T11:46:59 -02:00","latitude":-66.827751,"longitude":99.220079,"tags":["wontfix","bug","new issue","new issue"]} +{"id":15,"isActive":false,"balance":"$2,698.16","picture":"http://placehold.it/32x32","age":28,"color":"blue","name":"Aurelia Contreras","gender":"female","email":"aureliacontreras@chorizon.com","phone":"+1 (932) 442-3103","address":"655 Dwight Street, Grapeview, Palau, 8356","about":"Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n","registered":"2014-09-11T10:43:15 -02:00","latitude":-71.328973,"longitude":133.404895,"tags":["wontfix","bug","good first issue"]} +{"id":16,"isActive":true,"balance":"$3,303.25","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Estella Bass","gender":"female","email":"estellabass@chorizon.com","phone":"+1 (825) 436-2909","address":"435 Rockwell Place, Garberville, Wisconsin, 2230","about":"Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n","registered":"2017-11-23T09:32:09 -01:00","latitude":81.17014,"longitude":-145.262693,"tags":["new issue"]} +{"id":17,"isActive":false,"balance":"$3,579.20","picture":"http://placehold.it/32x32","age":25,"color":"brown","name":"Ortega Brennan","gender":"male","email":"ortegabrennan@chorizon.com","phone":"+1 (906) 526-2287","address":"440 Berry Street, Rivera, Maine, 1849","about":"Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n","registered":"2016-03-31T02:17:13 -02:00","latitude":-68.407524,"longitude":-113.642067,"tags":["new issue","wontfix"]} +{"id":18,"isActive":false,"balance":"$1,484.92","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Leonard Tillman","gender":"male","email":"leonardtillman@chorizon.com","phone":"+1 (864) 541-3456","address":"985 Provost Street, Charco, New Hampshire, 8632","about":"Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n","registered":"2018-05-06T08:21:27 -02:00","latitude":-8.581801,"longitude":-61.910062,"tags":["wontfix","new issue","bug","bug"]} +{"id":19,"isActive":true,"balance":"$3,572.55","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Dale Payne","gender":"male","email":"dalepayne@chorizon.com","phone":"+1 (814) 469-3499","address":"536 Dare Court, Ironton, Arkansas, 8605","about":"Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n","registered":"2019-10-11T01:01:33 -02:00","latitude":-18.280968,"longitude":-126.091797,"tags":["bug","wontfix","wontfix","wontfix"]} +{"id":20,"isActive":true,"balance":"$1,986.48","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Florence Long","gender":"female","email":"florencelong@chorizon.com","phone":"+1 (972) 557-3858","address":"519 Hendrickson Street, Templeton, Hawaii, 2389","about":"Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n","registered":"2016-05-02T09:18:59 -02:00","latitude":-27.110866,"longitude":-45.09445,"tags":[]} +{"id":21,"isActive":true,"balance":"$1,440.09","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Levy Whitley","gender":"male","email":"levywhitley@chorizon.com","phone":"+1 (911) 458-2411","address":"187 Thomas Street, Hachita, North Carolina, 2989","about":"Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n","registered":"2014-04-30T07:31:38 -02:00","latitude":-6.537315,"longitude":171.813536,"tags":["bug"]} +{"id":22,"isActive":false,"balance":"$2,938.57","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Bernard Mcfarland","gender":"male","email":"bernardmcfarland@chorizon.com","phone":"+1 (979) 442-3386","address":"409 Hall Street, Keyport, Federated States Of Micronesia, 7011","about":"Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n","registered":"2017-08-10T10:07:59 -02:00","latitude":63.766795,"longitude":68.177069,"tags":[]} +{"id":23,"isActive":true,"balance":"$1,678.49","picture":"http://placehold.it/32x32","age":31,"color":"brown","name":"Blanca Mcclain","gender":"female","email":"blancamcclain@chorizon.com","phone":"+1 (976) 439-2772","address":"176 Crooke Avenue, Valle, Virginia, 5373","about":"Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n","registered":"2015-10-12T11:57:28 -02:00","latitude":-8.944564,"longitude":-150.711709,"tags":["bug","wontfix","good first issue"]} +{"id":24,"isActive":true,"balance":"$2,276.87","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Espinoza Ford","gender":"male","email":"espinozaford@chorizon.com","phone":"+1 (945) 429-3975","address":"137 Bowery Street, Itmann, District Of Columbia, 1864","about":"Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n","registered":"2014-03-26T02:16:08 -01:00","latitude":-37.137666,"longitude":-51.811757,"tags":["wontfix","bug"]} +{"id":25,"isActive":true,"balance":"$3,973.43","picture":"http://placehold.it/32x32","age":29,"color":"Green","name":"Sykes Conley","gender":"male","email":"sykesconley@chorizon.com","phone":"+1 (851) 401-3916","address":"345 Grand Street, Woodlands, Missouri, 4461","about":"Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n","registered":"2015-09-12T06:03:56 -02:00","latitude":67.282955,"longitude":-64.341323,"tags":["wontfix"]} +{"id":26,"isActive":false,"balance":"$1,431.50","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Barlow Duran","gender":"male","email":"barlowduran@chorizon.com","phone":"+1 (995) 436-2562","address":"481 Everett Avenue, Allison, Nebraska, 3065","about":"Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n","registered":"2017-06-29T04:28:43 -02:00","latitude":-38.70606,"longitude":55.02816,"tags":["new issue"]} +{"id":27,"isActive":true,"balance":"$3,478.27","picture":"http://placehold.it/32x32","age":31,"color":"blue","name":"Schwartz Morgan","gender":"male","email":"schwartzmorgan@chorizon.com","phone":"+1 (861) 507-2067","address":"451 Lincoln Road, Fairlee, Washington, 2717","about":"Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n","registered":"2016-05-10T08:34:54 -02:00","latitude":-75.886403,"longitude":93.044471,"tags":["bug","bug","wontfix","wontfix"]} +{"id":28,"isActive":true,"balance":"$2,825.59","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Kristy Leon","gender":"female","email":"kristyleon@chorizon.com","phone":"+1 (948) 465-2563","address":"594 Macon Street, Floris, South Dakota, 3565","about":"Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n","registered":"2014-12-14T04:10:29 -01:00","latitude":-50.01615,"longitude":-68.908804,"tags":["wontfix","good first issue"]} +{"id":29,"isActive":false,"balance":"$3,028.03","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Ashley Pittman","gender":"male","email":"ashleypittman@chorizon.com","phone":"+1 (928) 507-3523","address":"646 Adelphi Street, Clara, Colorado, 6056","about":"Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n","registered":"2016-01-07T10:40:48 -01:00","latitude":-58.766037,"longitude":-124.828485,"tags":["wontfix"]} +{"id":30,"isActive":true,"balance":"$2,021.11","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Stacy Espinoza","gender":"female","email":"stacyespinoza@chorizon.com","phone":"+1 (999) 487-3253","address":"931 Alabama Avenue, Bangor, Alaska, 8215","about":"Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n","registered":"2014-07-16T06:15:53 -02:00","latitude":41.560197,"longitude":177.697,"tags":["new issue","new issue","bug"]} +{"id":31,"isActive":false,"balance":"$3,609.82","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Vilma Garza","gender":"female","email":"vilmagarza@chorizon.com","phone":"+1 (944) 585-2021","address":"565 Tech Place, Sedley, Puerto Rico, 858","about":"Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n","registered":"2017-06-30T07:43:52 -02:00","latitude":-12.574889,"longitude":-54.771186,"tags":["new issue","wontfix","wontfix"]} +{"id":32,"isActive":false,"balance":"$2,882.34","picture":"http://placehold.it/32x32","age":38,"color":"brown","name":"June Dunlap","gender":"female","email":"junedunlap@chorizon.com","phone":"+1 (997) 504-2937","address":"353 Cozine Avenue, Goodville, Indiana, 1438","about":"Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n","registered":"2016-08-23T08:54:11 -02:00","latitude":-27.883363,"longitude":-163.919683,"tags":["new issue","new issue","bug","wontfix"]} +{"id":33,"isActive":true,"balance":"$3,556.54","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Cecilia Greer","gender":"female","email":"ceciliagreer@chorizon.com","phone":"+1 (977) 573-3498","address":"696 Withers Street, Lydia, Oklahoma, 3220","about":"Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n","registered":"2017-01-13T11:30:12 -01:00","latitude":60.467215,"longitude":84.684575,"tags":["wontfix","good first issue","good first issue","wontfix"]} +{"id":34,"isActive":true,"balance":"$1,413.35","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Mckay Schroeder","gender":"male","email":"mckayschroeder@chorizon.com","phone":"+1 (816) 480-3657","address":"958 Miami Court, Rehrersburg, Northern Mariana Islands, 567","about":"Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n","registered":"2016-02-08T04:50:15 -01:00","latitude":-72.413287,"longitude":-159.254371,"tags":["good first issue"]} +{"id":35,"isActive":true,"balance":"$2,306.53","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Sawyer Mccormick","gender":"male","email":"sawyermccormick@chorizon.com","phone":"+1 (829) 569-3012","address":"749 Apollo Street, Eastvale, Texas, 7373","about":"Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n","registered":"2019-11-30T11:53:23 -01:00","latitude":-48.978194,"longitude":110.950191,"tags":["good first issue","new issue","new issue","bug"]} +{"id":36,"isActive":false,"balance":"$1,844.54","picture":"http://placehold.it/32x32","age":37,"color":"brown","name":"Barbra Valenzuela","gender":"female","email":"barbravalenzuela@chorizon.com","phone":"+1 (992) 512-2649","address":"617 Schenck Court, Reinerton, Michigan, 2908","about":"Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n","registered":"2019-03-29T01:59:31 -01:00","latitude":45.193723,"longitude":-12.486778,"tags":["new issue","new issue","wontfix","wontfix"]} +{"id":37,"isActive":false,"balance":"$3,469.82","picture":"http://placehold.it/32x32","age":39,"color":"brown","name":"Opal Weiss","gender":"female","email":"opalweiss@chorizon.com","phone":"+1 (809) 400-3079","address":"535 Bogart Street, Frizzleburg, Arizona, 5222","about":"Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n","registered":"2019-09-04T07:22:28 -02:00","latitude":72.50376,"longitude":61.656435,"tags":["bug","bug","good first issue","good first issue"]} +{"id":38,"isActive":true,"balance":"$1,992.38","picture":"http://placehold.it/32x32","age":40,"color":"Green","name":"Christina Short","gender":"female","email":"christinashort@chorizon.com","phone":"+1 (884) 589-2705","address":"594 Willmohr Street, Dexter, Montana, 660","about":"Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n","registered":"2014-01-21T09:31:56 -01:00","latitude":-42.762739,"longitude":77.052349,"tags":["bug","new issue"]} +{"id":39,"isActive":false,"balance":"$1,722.85","picture":"http://placehold.it/32x32","age":29,"color":"brown","name":"Golden Horton","gender":"male","email":"goldenhorton@chorizon.com","phone":"+1 (903) 426-2489","address":"191 Schenck Avenue, Mayfair, North Dakota, 5000","about":"Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n","registered":"2015-08-19T02:56:41 -02:00","latitude":69.922534,"longitude":9.881433,"tags":["bug"]} +{"id":40,"isActive":false,"balance":"$1,656.54","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Stafford Emerson","gender":"male","email":"staffordemerson@chorizon.com","phone":"+1 (992) 455-2573","address":"523 Thornton Street, Conway, Vermont, 6331","about":"Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n","registered":"2019-02-16T04:07:08 -01:00","latitude":-29.143111,"longitude":-57.207703,"tags":["wontfix","good first issue","good first issue"]} +{"id":41,"isActive":false,"balance":"$1,861.56","picture":"http://placehold.it/32x32","age":21,"color":"brown","name":"Salinas Gamble","gender":"male","email":"salinasgamble@chorizon.com","phone":"+1 (901) 525-2373","address":"991 Nostrand Avenue, Kansas, Mississippi, 6756","about":"Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n","registered":"2017-08-21T05:47:53 -02:00","latitude":-22.593819,"longitude":-63.613004,"tags":["good first issue","bug","bug","wontfix"]} +{"id":42,"isActive":true,"balance":"$3,179.74","picture":"http://placehold.it/32x32","age":34,"color":"brown","name":"Graciela Russell","gender":"female","email":"gracielarussell@chorizon.com","phone":"+1 (893) 464-3951","address":"361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713","about":"Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n","registered":"2015-05-18T09:52:56 -02:00","latitude":-14.634444,"longitude":12.931783,"tags":["wontfix","bug","wontfix"]} +{"id":43,"isActive":true,"balance":"$1,777.38","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Arnold Bender","gender":"male","email":"arnoldbender@chorizon.com","phone":"+1 (945) 581-3808","address":"781 Lorraine Street, Gallina, American Samoa, 1832","about":"Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n","registered":"2018-12-23T02:26:30 -01:00","latitude":41.208579,"longitude":51.948925,"tags":["bug","good first issue","good first issue","wontfix"]} +{"id":44,"isActive":true,"balance":"$2,893.45","picture":"http://placehold.it/32x32","age":22,"color":"Green","name":"Joni Spears","gender":"female","email":"jonispears@chorizon.com","phone":"+1 (916) 565-2124","address":"307 Harwood Place, Canterwood, Maryland, 2047","about":"Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n","registered":"2015-03-01T12:38:28 -01:00","latitude":8.19071,"longitude":146.323808,"tags":["wontfix","new issue","good first issue","good first issue"]} +{"id":45,"isActive":true,"balance":"$2,830.36","picture":"http://placehold.it/32x32","age":20,"color":"brown","name":"Irene Bennett","gender":"female","email":"irenebennett@chorizon.com","phone":"+1 (904) 431-2211","address":"353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686","about":"Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n","registered":"2018-04-17T05:18:51 -02:00","latitude":-36.435177,"longitude":-127.552573,"tags":["bug","wontfix"]} +{"id":46,"isActive":true,"balance":"$1,348.04","picture":"http://placehold.it/32x32","age":34,"color":"Green","name":"Lawson Curtis","gender":"male","email":"lawsoncurtis@chorizon.com","phone":"+1 (896) 532-2172","address":"942 Gerritsen Avenue, Southmont, Kansas, 8915","about":"Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n","registered":"2016-08-23T01:41:09 -02:00","latitude":-48.783539,"longitude":20.492944,"tags":[]} +{"id":47,"isActive":true,"balance":"$1,132.41","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goff May","gender":"male","email":"goffmay@chorizon.com","phone":"+1 (859) 453-3415","address":"225 Rutledge Street, Boonville, Massachusetts, 4081","about":"Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n","registered":"2014-10-25T07:32:30 -02:00","latitude":13.079225,"longitude":76.215086,"tags":["bug"]} +{"id":48,"isActive":true,"balance":"$1,201.87","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goodman Becker","gender":"male","email":"goodmanbecker@chorizon.com","phone":"+1 (825) 470-3437","address":"388 Seigel Street, Sisquoc, Kentucky, 8231","about":"Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n","registered":"2019-09-05T04:49:03 -02:00","latitude":-23.792094,"longitude":-13.621221,"tags":["bug","bug","wontfix","new issue"]} +{"id":49,"isActive":true,"balance":"$1,476.39","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Maureen Dale","gender":"female","email":"maureendale@chorizon.com","phone":"+1 (984) 538-3684","address":"817 Newton Street, Bannock, Wyoming, 1468","about":"Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n","registered":"2018-04-26T06:04:40 -02:00","latitude":-64.196802,"longitude":-117.396238,"tags":["wontfix"]} +{"id":50,"isActive":true,"balance":"$1,947.08","picture":"http://placehold.it/32x32","age":21,"color":"Green","name":"Guerra Mcintyre","gender":"male","email":"guerramcintyre@chorizon.com","phone":"+1 (951) 536-2043","address":"423 Lombardy Street, Stewart, West Virginia, 908","about":"Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n","registered":"2015-07-16T05:11:42 -02:00","latitude":79.733743,"longitude":-20.602356,"tags":["bug","good first issue","good first issue"]} +{"id":51,"isActive":true,"balance":"$2,960.90","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Key Cervantes","gender":"male","email":"keycervantes@chorizon.com","phone":"+1 (931) 474-3865","address":"410 Barbey Street, Vernon, Oregon, 2328","about":"Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n","registered":"2019-12-15T12:13:35 -01:00","latitude":47.627647,"longitude":117.049918,"tags":["new issue"]} +{"id":52,"isActive":false,"balance":"$1,884.02","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Karen Nelson","gender":"female","email":"karennelson@chorizon.com","phone":"+1 (993) 528-3607","address":"930 Frank Court, Dunbar, New York, 8810","about":"Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n","registered":"2014-06-23T09:21:44 -02:00","latitude":-59.059033,"longitude":76.565373,"tags":["new issue","bug"]} +{"id":53,"isActive":true,"balance":"$3,559.55","picture":"http://placehold.it/32x32","age":32,"color":"brown","name":"Caitlin Burnett","gender":"female","email":"caitlinburnett@chorizon.com","phone":"+1 (945) 480-2796","address":"516 Senator Street, Emory, Iowa, 4145","about":"In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n","registered":"2019-01-09T02:26:31 -01:00","latitude":-82.774237,"longitude":42.316194,"tags":["bug","good first issue"]} +{"id":54,"isActive":true,"balance":"$2,113.29","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Richards Walls","gender":"male","email":"richardswalls@chorizon.com","phone":"+1 (865) 517-2982","address":"959 Brightwater Avenue, Stevens, Nevada, 2968","about":"Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n","registered":"2014-09-25T06:51:22 -02:00","latitude":80.09202,"longitude":87.49759,"tags":["wontfix","wontfix","bug"]} +{"id":55,"isActive":true,"balance":"$1,977.66","picture":"http://placehold.it/32x32","age":36,"color":"brown","name":"Combs Stanley","gender":"male","email":"combsstanley@chorizon.com","phone":"+1 (827) 419-2053","address":"153 Beverley Road, Siglerville, South Carolina, 3666","about":"Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n","registered":"2019-08-22T07:53:15 -02:00","latitude":78.386181,"longitude":143.661058,"tags":[]} +{"id":56,"isActive":false,"balance":"$3,886.12","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Tucker Barry","gender":"male","email":"tuckerbarry@chorizon.com","phone":"+1 (808) 544-3433","address":"805 Jamaica Avenue, Cornfields, Minnesota, 3689","about":"Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n","registered":"2016-08-29T07:28:00 -02:00","latitude":71.701551,"longitude":9.903068,"tags":[]} +{"id":57,"isActive":false,"balance":"$1,844.56","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Kaitlin Conner","gender":"female","email":"kaitlinconner@chorizon.com","phone":"+1 (862) 467-2666","address":"501 Knight Court, Joppa, Rhode Island, 274","about":"Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n","registered":"2019-05-30T06:38:24 -02:00","latitude":15.613464,"longitude":171.965629,"tags":[]} +{"id":58,"isActive":true,"balance":"$2,876.10","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Mamie Fischer","gender":"female","email":"mamiefischer@chorizon.com","phone":"+1 (948) 545-3901","address":"599 Hunterfly Place, Haena, Georgia, 6005","about":"Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n","registered":"2019-05-27T05:07:10 -02:00","latitude":70.915079,"longitude":-48.813584,"tags":["bug","wontfix","wontfix","good first issue"]} +{"id":59,"isActive":true,"balance":"$1,921.58","picture":"http://placehold.it/32x32","age":31,"color":"Green","name":"Harper Carson","gender":"male","email":"harpercarson@chorizon.com","phone":"+1 (912) 430-3243","address":"883 Dennett Place, Knowlton, New Mexico, 9219","about":"Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n","registered":"2019-12-07T07:33:15 -01:00","latitude":-60.812605,"longitude":-27.129016,"tags":["bug","new issue"]} +{"id":60,"isActive":true,"balance":"$1,770.93","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Jody Herrera","gender":"female","email":"jodyherrera@chorizon.com","phone":"+1 (890) 583-3222","address":"261 Jay Street, Strykersville, Ohio, 9248","about":"Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n","registered":"2016-05-21T01:00:02 -02:00","latitude":-36.846586,"longitude":131.156223,"tags":[]} +{"id":61,"isActive":false,"balance":"$2,813.41","picture":"http://placehold.it/32x32","age":37,"color":"Green","name":"Charles Castillo","gender":"male","email":"charlescastillo@chorizon.com","phone":"+1 (934) 467-2108","address":"675 Morton Street, Rew, Pennsylvania, 137","about":"Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n","registered":"2019-06-10T02:54:22 -02:00","latitude":-16.423202,"longitude":-146.293752,"tags":["new issue","new issue"]} +{"id":62,"isActive":true,"balance":"$3,341.35","picture":"http://placehold.it/32x32","age":33,"color":"blue","name":"Estelle Ramirez","gender":"female","email":"estelleramirez@chorizon.com","phone":"+1 (816) 459-2073","address":"636 Nolans Lane, Camptown, California, 7794","about":"Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n","registered":"2015-02-14T01:05:50 -01:00","latitude":-46.591249,"longitude":-83.385587,"tags":["good first issue","bug"]} +{"id":63,"isActive":true,"balance":"$2,478.30","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Knowles Hebert","gender":"male","email":"knowleshebert@chorizon.com","phone":"+1 (819) 409-2308","address":"361 Kathleen Court, Gratton, Connecticut, 7254","about":"Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n","registered":"2016-03-08T08:34:52 -01:00","latitude":71.042482,"longitude":152.460406,"tags":["good first issue","wontfix"]} +{"id":64,"isActive":false,"balance":"$2,559.09","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Thelma Mckenzie","gender":"female","email":"thelmamckenzie@chorizon.com","phone":"+1 (941) 596-2777","address":"202 Leonard Street, Riverton, Illinois, 8577","about":"Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n","registered":"2020-04-14T12:43:06 -02:00","latitude":16.026129,"longitude":105.464476,"tags":[]} +{"id":65,"isActive":true,"balance":"$1,025.08","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Carole Rowland","gender":"female","email":"carolerowland@chorizon.com","phone":"+1 (862) 558-3448","address":"941 Melba Court, Bluetown, Florida, 9555","about":"Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n","registered":"2014-12-01T05:55:35 -01:00","latitude":-0.191998,"longitude":43.389652,"tags":["wontfix"]} +{"id":66,"isActive":true,"balance":"$1,061.49","picture":"http://placehold.it/32x32","age":35,"color":"brown","name":"Higgins Aguilar","gender":"male","email":"higginsaguilar@chorizon.com","phone":"+1 (911) 540-3791","address":"132 Sackman Street, Layhill, Guam, 8729","about":"Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n","registered":"2015-04-05T02:10:07 -02:00","latitude":74.702813,"longitude":151.314972,"tags":["bug"]} +{"id":67,"isActive":true,"balance":"$3,510.14","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Ilene Gillespie","gender":"female","email":"ilenegillespie@chorizon.com","phone":"+1 (937) 575-2676","address":"835 Lake Street, Naomi, Alabama, 4131","about":"Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n","registered":"2015-06-28T09:41:45 -02:00","latitude":71.573342,"longitude":-95.295989,"tags":["wontfix","wontfix"]} +{"id":68,"isActive":false,"balance":"$1,539.98","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Angelina Dyer","gender":"female","email":"angelinadyer@chorizon.com","phone":"+1 (948) 574-3949","address":"575 Division Place, Gorham, Louisiana, 3458","about":"Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n","registered":"2014-07-08T06:34:36 -02:00","latitude":-85.649593,"longitude":66.126018,"tags":["good first issue"]} +{"id":69,"isActive":true,"balance":"$3,367.69","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Marks Burt","gender":"male","email":"marksburt@chorizon.com","phone":"+1 (895) 497-3138","address":"819 Village Road, Wadsworth, Delaware, 6099","about":"Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n","registered":"2014-08-31T06:12:18 -02:00","latitude":26.854112,"longitude":-143.313948,"tags":["good first issue"]} +{"id":70,"isActive":false,"balance":"$3,755.72","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Glass Perkins","gender":"male","email":"glassperkins@chorizon.com","phone":"+1 (923) 486-3725","address":"899 Roosevelt Court, Belleview, Idaho, 1737","about":"Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n","registered":"2015-05-22T05:44:33 -02:00","latitude":54.27147,"longitude":-65.065604,"tags":["wontfix"]} +{"id":71,"isActive":true,"balance":"$3,381.63","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Candace Sawyer","gender":"female","email":"candacesawyer@chorizon.com","phone":"+1 (830) 404-2636","address":"334 Arkansas Drive, Bordelonville, Tennessee, 8449","about":"Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n","registered":"2014-04-04T08:45:00 -02:00","latitude":6.484262,"longitude":-37.054928,"tags":["new issue","new issue"]} +{"id":72,"isActive":true,"balance":"$1,640.98","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Hendricks Martinez","gender":"male","email":"hendricksmartinez@chorizon.com","phone":"+1 (857) 566-3245","address":"636 Agate Court, Newry, Utah, 3304","about":"Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n","registered":"2018-06-15T10:36:11 -02:00","latitude":86.746034,"longitude":10.347893,"tags":["new issue"]} +{"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]} +{"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]} +{"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]} +{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]} \ No newline at end of file diff --git a/meilisearch-http/tests/assets/dumps/v1/test/settings.json b/meilisearch-http/tests/assets/dumps/v1/test/settings.json new file mode 100644 index 000000000..918cfab53 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/settings.json @@ -0,0 +1,59 @@ +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": "email", + "searchableAttributes": [ + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "displayedAttributes": [ + "id", + "isActive", + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "stopWords": [ + "in", + "ad" + ], + "synonyms": { + "wolverine": ["xmen", "logan"], + "logan": ["wolverine", "xmen"] + }, + "attributesForFaceting": [ + "gender", + "color", + "tags", + "isActive" + ] +} diff --git a/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl new file mode 100644 index 000000000..0dcffdce0 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl @@ -0,0 +1,2 @@ +{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}} +{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} \ No newline at end of file diff --git a/meilisearch-http/tests/assets/test_set.json b/meilisearch-http/tests/assets/test_set.json new file mode 100644 index 000000000..63534c896 --- /dev/null +++ b/meilisearch-http/tests/assets/test_set.json @@ -0,0 +1,1613 @@ +[ + { + "id": 0, + "isActive": false, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ] + }, + { + "id": 1, + "isActive": true, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ] + }, + { + "id": 2, + "isActive": true, + "balance": "$2,467.47", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", + "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", + "registered": "2014-10-28T12:59:30 -01:00", + "latitude": -64.008555, + "longitude": 11.867098, + "tags": [ + "good first issue" + ] + }, + { + "id": 3, + "isActive": true, + "balance": "$3,344.40", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Adeline Flynn", + "gender": "female", + "email": "adelineflynn@chorizon.com", + "phone": "+1 (994) 600-2840", + "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948", + "about": "Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n", + "registered": "2014-03-27T06:24:45 -01:00", + "latitude": -74.485173, + "longitude": -11.059859, + "tags": [ + "bug", + "good first issue", + "wontfix", + "new issue" + ] + }, + { + "id": 4, + "isActive": false, + "balance": "$2,575.78", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "Green", + "name": "Mariana Pacheco", + "gender": "female", + "email": "marianapacheco@chorizon.com", + "phone": "+1 (820) 414-2223", + "address": "664 Rapelye Street, Faywood, California, 7320", + "about": "Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n", + "registered": "2015-09-02T03:23:35 -02:00", + "latitude": 75.763501, + "longitude": -78.777124, + "tags": [ + "new issue" + ] + }, + { + "id": 5, + "isActive": true, + "balance": "$3,793.09", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "Green", + "name": "Warren Watson", + "gender": "male", + "email": "warrenwatson@chorizon.com", + "phone": "+1 (807) 583-2427", + "address": "671 Prince Street, Faxon, Connecticut, 4275", + "about": "Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n", + "registered": "2017-06-04T06:02:17 -02:00", + "latitude": 29.979223, + "longitude": 25.358943, + "tags": [ + "wontfix", + "wontfix", + "wontfix" + ] + }, + { + "id": 6, + "isActive": true, + "balance": "$2,919.70", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "blue", + "name": "Shelia Berry", + "gender": "female", + "email": "sheliaberry@chorizon.com", + "phone": "+1 (853) 511-2651", + "address": "437 Forrest Street, Coventry, Illinois, 2056", + "about": "Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n", + "registered": "2018-07-11T02:45:01 -02:00", + "latitude": 54.815991, + "longitude": -118.690609, + "tags": [ + "good first issue", + "bug", + "wontfix", + "new issue" + ] + }, + { + "id": 7, + "isActive": true, + "balance": "$1,349.50", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Chrystal Boyd", + "gender": "female", + "email": "chrystalboyd@chorizon.com", + "phone": "+1 (936) 563-2802", + "address": "670 Croton Loop, Sussex, Florida, 4692", + "about": "Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n", + "registered": "2016-11-01T07:36:04 -01:00", + "latitude": -24.711933, + "longitude": 147.246705, + "tags": [] + }, + { + "id": 8, + "isActive": false, + "balance": "$3,999.56", + "picture": "http://placehold.it/32x32", + "age": 30, + "color": "brown", + "name": "Martin Porter", + "gender": "male", + "email": "martinporter@chorizon.com", + "phone": "+1 (895) 580-2304", + "address": "577 Regent Place, Aguila, Guam, 6554", + "about": "Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n", + "registered": "2014-09-20T02:08:30 -02:00", + "latitude": -88.344273, + "longitude": 37.964466, + "tags": [] + }, + { + "id": 9, + "isActive": true, + "balance": "$3,729.71", + "picture": "http://placehold.it/32x32", + "age": 26, + "color": "blue", + "name": "Kelli Mendez", + "gender": "female", + "email": "kellimendez@chorizon.com", + "phone": "+1 (936) 401-2236", + "address": "242 Caton Place, Grazierville, Alabama, 3968", + "about": "Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n", + "registered": "2018-05-04T10:35:30 -02:00", + "latitude": 49.37551, + "longitude": 41.872323, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 10, + "isActive": false, + "balance": "$1,127.47", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "blue", + "name": "Maddox Johns", + "gender": "male", + "email": "maddoxjohns@chorizon.com", + "phone": "+1 (892) 470-2357", + "address": "756 Beard Street, Avalon, Louisiana, 114", + "about": "Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n", + "registered": "2016-04-22T06:41:25 -02:00", + "latitude": 66.640229, + "longitude": -17.222666, + "tags": [ + "new issue", + "good first issue", + "good first issue", + "new issue" + ] + }, + { + "id": 11, + "isActive": true, + "balance": "$1,351.43", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Evans Wagner", + "gender": "male", + "email": "evanswagner@chorizon.com", + "phone": "+1 (889) 496-2332", + "address": "118 Monaco Place, Lutsen, Delaware, 6209", + "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", + "registered": "2016-10-27T01:26:31 -02:00", + "latitude": -77.673222, + "longitude": -142.657214, + "tags": [ + "good first issue", + "good first issue" + ] + }, + { + "id": 12, + "isActive": false, + "balance": "$3,394.96", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "blue", + "name": "Aida Kirby", + "gender": "female", + "email": "aidakirby@chorizon.com", + "phone": "+1 (942) 532-2325", + "address": "797 Engert Avenue, Wilsonia, Idaho, 6532", + "about": "Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n", + "registered": "2018-06-18T04:39:57 -02:00", + "latitude": -58.062041, + "longitude": 34.999254, + "tags": [ + "new issue", + "wontfix", + "bug", + "new issue" + ] + }, + { + "id": 13, + "isActive": true, + "balance": "$2,812.62", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "blue", + "name": "Nelda Burris", + "gender": "female", + "email": "neldaburris@chorizon.com", + "phone": "+1 (813) 600-2576", + "address": "160 Opal Court, Fowlerville, Tennessee, 2170", + "about": "Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n", + "registered": "2015-08-15T12:39:53 -02:00", + "latitude": 66.6871, + "longitude": 179.549488, + "tags": [ + "wontfix" + ] + }, + { + "id": 14, + "isActive": true, + "balance": "$1,718.33", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Jennifer Hart", + "gender": "female", + "email": "jenniferhart@chorizon.com", + "phone": "+1 (850) 537-2513", + "address": "124 Veranda Place, Nash, Utah, 985", + "about": "Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n", + "registered": "2016-09-04T11:46:59 -02:00", + "latitude": -66.827751, + "longitude": 99.220079, + "tags": [ + "wontfix", + "bug", + "new issue", + "new issue" + ] + }, + { + "id": 15, + "isActive": false, + "balance": "$2,698.16", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "blue", + "name": "Aurelia Contreras", + "gender": "female", + "email": "aureliacontreras@chorizon.com", + "phone": "+1 (932) 442-3103", + "address": "655 Dwight Street, Grapeview, Palau, 8356", + "about": "Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n", + "registered": "2014-09-11T10:43:15 -02:00", + "latitude": -71.328973, + "longitude": 133.404895, + "tags": [ + "wontfix", + "bug", + "good first issue" + ] + }, + { + "id": 16, + "isActive": true, + "balance": "$3,303.25", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Estella Bass", + "gender": "female", + "email": "estellabass@chorizon.com", + "phone": "+1 (825) 436-2909", + "address": "435 Rockwell Place, Garberville, Wisconsin, 2230", + "about": "Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n", + "registered": "2017-11-23T09:32:09 -01:00", + "latitude": 81.17014, + "longitude": -145.262693, + "tags": [ + "new issue" + ] + }, + { + "id": 17, + "isActive": false, + "balance": "$3,579.20", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "brown", + "name": "Ortega Brennan", + "gender": "male", + "email": "ortegabrennan@chorizon.com", + "phone": "+1 (906) 526-2287", + "address": "440 Berry Street, Rivera, Maine, 1849", + "about": "Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n", + "registered": "2016-03-31T02:17:13 -02:00", + "latitude": -68.407524, + "longitude": -113.642067, + "tags": [ + "new issue", + "wontfix" + ] + }, + { + "id": 18, + "isActive": false, + "balance": "$1,484.92", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "blue", + "name": "Leonard Tillman", + "gender": "male", + "email": "leonardtillman@chorizon.com", + "phone": "+1 (864) 541-3456", + "address": "985 Provost Street, Charco, New Hampshire, 8632", + "about": "Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n", + "registered": "2018-05-06T08:21:27 -02:00", + "latitude": -8.581801, + "longitude": -61.910062, + "tags": [ + "wontfix", + "new issue", + "bug", + "bug" + ] + }, + { + "id": 19, + "isActive": true, + "balance": "$3,572.55", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Dale Payne", + "gender": "male", + "email": "dalepayne@chorizon.com", + "phone": "+1 (814) 469-3499", + "address": "536 Dare Court, Ironton, Arkansas, 8605", + "about": "Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n", + "registered": "2019-10-11T01:01:33 -02:00", + "latitude": -18.280968, + "longitude": -126.091797, + "tags": [ + "bug", + "wontfix", + "wontfix", + "wontfix" + ] + }, + { + "id": 20, + "isActive": true, + "balance": "$1,986.48", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Florence Long", + "gender": "female", + "email": "florencelong@chorizon.com", + "phone": "+1 (972) 557-3858", + "address": "519 Hendrickson Street, Templeton, Hawaii, 2389", + "about": "Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n", + "registered": "2016-05-02T09:18:59 -02:00", + "latitude": -27.110866, + "longitude": -45.09445, + "tags": [] + }, + { + "id": 21, + "isActive": true, + "balance": "$1,440.09", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "blue", + "name": "Levy Whitley", + "gender": "male", + "email": "levywhitley@chorizon.com", + "phone": "+1 (911) 458-2411", + "address": "187 Thomas Street, Hachita, North Carolina, 2989", + "about": "Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n", + "registered": "2014-04-30T07:31:38 -02:00", + "latitude": -6.537315, + "longitude": 171.813536, + "tags": [ + "bug" + ] + }, + { + "id": 22, + "isActive": false, + "balance": "$2,938.57", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Bernard Mcfarland", + "gender": "male", + "email": "bernardmcfarland@chorizon.com", + "phone": "+1 (979) 442-3386", + "address": "409 Hall Street, Keyport, Federated States Of Micronesia, 7011", + "about": "Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n", + "registered": "2017-08-10T10:07:59 -02:00", + "latitude": 63.766795, + "longitude": 68.177069, + "tags": [] + }, + { + "id": 23, + "isActive": true, + "balance": "$1,678.49", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "brown", + "name": "Blanca Mcclain", + "gender": "female", + "email": "blancamcclain@chorizon.com", + "phone": "+1 (976) 439-2772", + "address": "176 Crooke Avenue, Valle, Virginia, 5373", + "about": "Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n", + "registered": "2015-10-12T11:57:28 -02:00", + "latitude": -8.944564, + "longitude": -150.711709, + "tags": [ + "bug", + "wontfix", + "good first issue" + ] + }, + { + "id": 24, + "isActive": true, + "balance": "$2,276.87", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Espinoza Ford", + "gender": "male", + "email": "espinozaford@chorizon.com", + "phone": "+1 (945) 429-3975", + "address": "137 Bowery Street, Itmann, District Of Columbia, 1864", + "about": "Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n", + "registered": "2014-03-26T02:16:08 -01:00", + "latitude": -37.137666, + "longitude": -51.811757, + "tags": [ + "wontfix", + "bug" + ] + }, + { + "id": 25, + "isActive": true, + "balance": "$3,973.43", + "picture": "http://placehold.it/32x32", + "age": 29, + "color": "Green", + "name": "Sykes Conley", + "gender": "male", + "email": "sykesconley@chorizon.com", + "phone": "+1 (851) 401-3916", + "address": "345 Grand Street, Woodlands, Missouri, 4461", + "about": "Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n", + "registered": "2015-09-12T06:03:56 -02:00", + "latitude": 67.282955, + "longitude": -64.341323, + "tags": [ + "wontfix" + ] + }, + { + "id": 26, + "isActive": false, + "balance": "$1,431.50", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Barlow Duran", + "gender": "male", + "email": "barlowduran@chorizon.com", + "phone": "+1 (995) 436-2562", + "address": "481 Everett Avenue, Allison, Nebraska, 3065", + "about": "Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n", + "registered": "2017-06-29T04:28:43 -02:00", + "latitude": -38.70606, + "longitude": 55.02816, + "tags": [ + "new issue" + ] + }, + { + "id": 27, + "isActive": true, + "balance": "$3,478.27", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "blue", + "name": "Schwartz Morgan", + "gender": "male", + "email": "schwartzmorgan@chorizon.com", + "phone": "+1 (861) 507-2067", + "address": "451 Lincoln Road, Fairlee, Washington, 2717", + "about": "Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n", + "registered": "2016-05-10T08:34:54 -02:00", + "latitude": -75.886403, + "longitude": 93.044471, + "tags": [ + "bug", + "bug", + "wontfix", + "wontfix" + ] + }, + { + "id": 28, + "isActive": true, + "balance": "$2,825.59", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Kristy Leon", + "gender": "female", + "email": "kristyleon@chorizon.com", + "phone": "+1 (948) 465-2563", + "address": "594 Macon Street, Floris, South Dakota, 3565", + "about": "Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n", + "registered": "2014-12-14T04:10:29 -01:00", + "latitude": -50.01615, + "longitude": -68.908804, + "tags": [ + "wontfix", + "good first issue" + ] + }, + { + "id": 29, + "isActive": false, + "balance": "$3,028.03", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "blue", + "name": "Ashley Pittman", + "gender": "male", + "email": "ashleypittman@chorizon.com", + "phone": "+1 (928) 507-3523", + "address": "646 Adelphi Street, Clara, Colorado, 6056", + "about": "Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n", + "registered": "2016-01-07T10:40:48 -01:00", + "latitude": -58.766037, + "longitude": -124.828485, + "tags": [ + "wontfix" + ] + }, + { + "id": 30, + "isActive": true, + "balance": "$2,021.11", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Stacy Espinoza", + "gender": "female", + "email": "stacyespinoza@chorizon.com", + "phone": "+1 (999) 487-3253", + "address": "931 Alabama Avenue, Bangor, Alaska, 8215", + "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", + "registered": "2014-07-16T06:15:53 -02:00", + "latitude": 41.560197, + "longitude": 177.697, + "tags": [ + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 31, + "isActive": false, + "balance": "$3,609.82", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Vilma Garza", + "gender": "female", + "email": "vilmagarza@chorizon.com", + "phone": "+1 (944) 585-2021", + "address": "565 Tech Place, Sedley, Puerto Rico, 858", + "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", + "registered": "2017-06-30T07:43:52 -02:00", + "latitude": -12.574889, + "longitude": -54.771186, + "tags": [ + "new issue", + "wontfix", + "wontfix" + ] + }, + { + "id": 32, + "isActive": false, + "balance": "$2,882.34", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "brown", + "name": "June Dunlap", + "gender": "female", + "email": "junedunlap@chorizon.com", + "phone": "+1 (997) 504-2937", + "address": "353 Cozine Avenue, Goodville, Indiana, 1438", + "about": "Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n", + "registered": "2016-08-23T08:54:11 -02:00", + "latitude": -27.883363, + "longitude": -163.919683, + "tags": [ + "new issue", + "new issue", + "bug", + "wontfix" + ] + }, + { + "id": 33, + "isActive": true, + "balance": "$3,556.54", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Cecilia Greer", + "gender": "female", + "email": "ceciliagreer@chorizon.com", + "phone": "+1 (977) 573-3498", + "address": "696 Withers Street, Lydia, Oklahoma, 3220", + "about": "Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n", + "registered": "2017-01-13T11:30:12 -01:00", + "latitude": 60.467215, + "longitude": 84.684575, + "tags": [ + "wontfix", + "good first issue", + "good first issue", + "wontfix" + ] + }, + { + "id": 34, + "isActive": true, + "balance": "$1,413.35", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Mckay Schroeder", + "gender": "male", + "email": "mckayschroeder@chorizon.com", + "phone": "+1 (816) 480-3657", + "address": "958 Miami Court, Rehrersburg, Northern Mariana Islands, 567", + "about": "Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n", + "registered": "2016-02-08T04:50:15 -01:00", + "latitude": -72.413287, + "longitude": -159.254371, + "tags": [ + "good first issue" + ] + }, + { + "id": 35, + "isActive": true, + "balance": "$2,306.53", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Sawyer Mccormick", + "gender": "male", + "email": "sawyermccormick@chorizon.com", + "phone": "+1 (829) 569-3012", + "address": "749 Apollo Street, Eastvale, Texas, 7373", + "about": "Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n", + "registered": "2019-11-30T11:53:23 -01:00", + "latitude": -48.978194, + "longitude": 110.950191, + "tags": [ + "good first issue", + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 36, + "isActive": false, + "balance": "$1,844.54", + "picture": "http://placehold.it/32x32", + "age": 37, + "color": "brown", + "name": "Barbra Valenzuela", + "gender": "female", + "email": "barbravalenzuela@chorizon.com", + "phone": "+1 (992) 512-2649", + "address": "617 Schenck Court, Reinerton, Michigan, 2908", + "about": "Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n", + "registered": "2019-03-29T01:59:31 -01:00", + "latitude": 45.193723, + "longitude": -12.486778, + "tags": [ + "new issue", + "new issue", + "wontfix", + "wontfix" + ] + }, + { + "id": 37, + "isActive": false, + "balance": "$3,469.82", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "brown", + "name": "Opal Weiss", + "gender": "female", + "email": "opalweiss@chorizon.com", + "phone": "+1 (809) 400-3079", + "address": "535 Bogart Street, Frizzleburg, Arizona, 5222", + "about": "Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n", + "registered": "2019-09-04T07:22:28 -02:00", + "latitude": 72.50376, + "longitude": 61.656435, + "tags": [ + "bug", + "bug", + "good first issue", + "good first issue" + ] + }, + { + "id": 38, + "isActive": true, + "balance": "$1,992.38", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "Green", + "name": "Christina Short", + "gender": "female", + "email": "christinashort@chorizon.com", + "phone": "+1 (884) 589-2705", + "address": "594 Willmohr Street, Dexter, Montana, 660", + "about": "Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n", + "registered": "2014-01-21T09:31:56 -01:00", + "latitude": -42.762739, + "longitude": 77.052349, + "tags": [ + "bug", + "new issue" + ] + }, + { + "id": 39, + "isActive": false, + "balance": "$1,722.85", + "picture": "http://placehold.it/32x32", + "age": 29, + "color": "brown", + "name": "Golden Horton", + "gender": "male", + "email": "goldenhorton@chorizon.com", + "phone": "+1 (903) 426-2489", + "address": "191 Schenck Avenue, Mayfair, North Dakota, 5000", + "about": "Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n", + "registered": "2015-08-19T02:56:41 -02:00", + "latitude": 69.922534, + "longitude": 9.881433, + "tags": [ + "bug" + ] + }, + { + "id": 40, + "isActive": false, + "balance": "$1,656.54", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "blue", + "name": "Stafford Emerson", + "gender": "male", + "email": "staffordemerson@chorizon.com", + "phone": "+1 (992) 455-2573", + "address": "523 Thornton Street, Conway, Vermont, 6331", + "about": "Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n", + "registered": "2019-02-16T04:07:08 -01:00", + "latitude": -29.143111, + "longitude": -57.207703, + "tags": [ + "wontfix", + "good first issue", + "good first issue" + ] + }, + { + "id": 41, + "isActive": false, + "balance": "$1,861.56", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "brown", + "name": "Salinas Gamble", + "gender": "male", + "email": "salinasgamble@chorizon.com", + "phone": "+1 (901) 525-2373", + "address": "991 Nostrand Avenue, Kansas, Mississippi, 6756", + "about": "Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n", + "registered": "2017-08-21T05:47:53 -02:00", + "latitude": -22.593819, + "longitude": -63.613004, + "tags": [ + "good first issue", + "bug", + "bug", + "wontfix" + ] + }, + { + "id": 42, + "isActive": true, + "balance": "$3,179.74", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "brown", + "name": "Graciela Russell", + "gender": "female", + "email": "gracielarussell@chorizon.com", + "phone": "+1 (893) 464-3951", + "address": "361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713", + "about": "Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n", + "registered": "2015-05-18T09:52:56 -02:00", + "latitude": -14.634444, + "longitude": 12.931783, + "tags": [ + "wontfix", + "bug", + "wontfix" + ] + }, + { + "id": 43, + "isActive": true, + "balance": "$1,777.38", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "blue", + "name": "Arnold Bender", + "gender": "male", + "email": "arnoldbender@chorizon.com", + "phone": "+1 (945) 581-3808", + "address": "781 Lorraine Street, Gallina, American Samoa, 1832", + "about": "Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n", + "registered": "2018-12-23T02:26:30 -01:00", + "latitude": 41.208579, + "longitude": 51.948925, + "tags": [ + "bug", + "good first issue", + "good first issue", + "wontfix" + ] + }, + { + "id": 44, + "isActive": true, + "balance": "$2,893.45", + "picture": "http://placehold.it/32x32", + "age": 22, + "color": "Green", + "name": "Joni Spears", + "gender": "female", + "email": "jonispears@chorizon.com", + "phone": "+1 (916) 565-2124", + "address": "307 Harwood Place, Canterwood, Maryland, 2047", + "about": "Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n", + "registered": "2015-03-01T12:38:28 -01:00", + "latitude": 8.19071, + "longitude": 146.323808, + "tags": [ + "wontfix", + "new issue", + "good first issue", + "good first issue" + ] + }, + { + "id": 45, + "isActive": true, + "balance": "$2,830.36", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "brown", + "name": "Irene Bennett", + "gender": "female", + "email": "irenebennett@chorizon.com", + "phone": "+1 (904) 431-2211", + "address": "353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686", + "about": "Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n", + "registered": "2018-04-17T05:18:51 -02:00", + "latitude": -36.435177, + "longitude": -127.552573, + "tags": [ + "bug", + "wontfix" + ] + }, + { + "id": 46, + "isActive": true, + "balance": "$1,348.04", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "Green", + "name": "Lawson Curtis", + "gender": "male", + "email": "lawsoncurtis@chorizon.com", + "phone": "+1 (896) 532-2172", + "address": "942 Gerritsen Avenue, Southmont, Kansas, 8915", + "about": "Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n", + "registered": "2016-08-23T01:41:09 -02:00", + "latitude": -48.783539, + "longitude": 20.492944, + "tags": [] + }, + { + "id": 47, + "isActive": true, + "balance": "$1,132.41", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Goff May", + "gender": "male", + "email": "goffmay@chorizon.com", + "phone": "+1 (859) 453-3415", + "address": "225 Rutledge Street, Boonville, Massachusetts, 4081", + "about": "Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n", + "registered": "2014-10-25T07:32:30 -02:00", + "latitude": 13.079225, + "longitude": 76.215086, + "tags": [ + "bug" + ] + }, + { + "id": 48, + "isActive": true, + "balance": "$1,201.87", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Goodman Becker", + "gender": "male", + "email": "goodmanbecker@chorizon.com", + "phone": "+1 (825) 470-3437", + "address": "388 Seigel Street, Sisquoc, Kentucky, 8231", + "about": "Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n", + "registered": "2019-09-05T04:49:03 -02:00", + "latitude": -23.792094, + "longitude": -13.621221, + "tags": [ + "bug", + "bug", + "wontfix", + "new issue" + ] + }, + { + "id": 49, + "isActive": true, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ] + }, + { + "id": 50, + "isActive": true, + "balance": "$1,947.08", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "Green", + "name": "Guerra Mcintyre", + "gender": "male", + "email": "guerramcintyre@chorizon.com", + "phone": "+1 (951) 536-2043", + "address": "423 Lombardy Street, Stewart, West Virginia, 908", + "about": "Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n", + "registered": "2015-07-16T05:11:42 -02:00", + "latitude": 79.733743, + "longitude": -20.602356, + "tags": [ + "bug", + "good first issue", + "good first issue" + ] + }, + { + "id": 51, + "isActive": true, + "balance": "$2,960.90", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "blue", + "name": "Key Cervantes", + "gender": "male", + "email": "keycervantes@chorizon.com", + "phone": "+1 (931) 474-3865", + "address": "410 Barbey Street, Vernon, Oregon, 2328", + "about": "Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n", + "registered": "2019-12-15T12:13:35 -01:00", + "latitude": 47.627647, + "longitude": 117.049918, + "tags": [ + "new issue" + ] + }, + { + "id": 52, + "isActive": false, + "balance": "$1,884.02", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Karen Nelson", + "gender": "female", + "email": "karennelson@chorizon.com", + "phone": "+1 (993) 528-3607", + "address": "930 Frank Court, Dunbar, New York, 8810", + "about": "Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n", + "registered": "2014-06-23T09:21:44 -02:00", + "latitude": -59.059033, + "longitude": 76.565373, + "tags": [ + "new issue", + "bug" + ] + }, + { + "id": 53, + "isActive": true, + "balance": "$3,559.55", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "brown", + "name": "Caitlin Burnett", + "gender": "female", + "email": "caitlinburnett@chorizon.com", + "phone": "+1 (945) 480-2796", + "address": "516 Senator Street, Emory, Iowa, 4145", + "about": "In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n", + "registered": "2019-01-09T02:26:31 -01:00", + "latitude": -82.774237, + "longitude": 42.316194, + "tags": [ + "bug", + "good first issue" + ] + }, + { + "id": 54, + "isActive": true, + "balance": "$2,113.29", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Richards Walls", + "gender": "male", + "email": "richardswalls@chorizon.com", + "phone": "+1 (865) 517-2982", + "address": "959 Brightwater Avenue, Stevens, Nevada, 2968", + "about": "Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n", + "registered": "2014-09-25T06:51:22 -02:00", + "latitude": 80.09202, + "longitude": 87.49759, + "tags": [ + "wontfix", + "wontfix", + "bug" + ] + }, + { + "id": 55, + "isActive": true, + "balance": "$1,977.66", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "brown", + "name": "Combs Stanley", + "gender": "male", + "email": "combsstanley@chorizon.com", + "phone": "+1 (827) 419-2053", + "address": "153 Beverley Road, Siglerville, South Carolina, 3666", + "about": "Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n", + "registered": "2019-08-22T07:53:15 -02:00", + "latitude": 78.386181, + "longitude": 143.661058, + "tags": [] + }, + { + "id": 56, + "isActive": false, + "balance": "$3,886.12", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "brown", + "name": "Tucker Barry", + "gender": "male", + "email": "tuckerbarry@chorizon.com", + "phone": "+1 (808) 544-3433", + "address": "805 Jamaica Avenue, Cornfields, Minnesota, 3689", + "about": "Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n", + "registered": "2016-08-29T07:28:00 -02:00", + "latitude": 71.701551, + "longitude": 9.903068, + "tags": [] + }, + { + "id": 57, + "isActive": false, + "balance": "$1,844.56", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "Green", + "name": "Kaitlin Conner", + "gender": "female", + "email": "kaitlinconner@chorizon.com", + "phone": "+1 (862) 467-2666", + "address": "501 Knight Court, Joppa, Rhode Island, 274", + "about": "Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n", + "registered": "2019-05-30T06:38:24 -02:00", + "latitude": 15.613464, + "longitude": 171.965629, + "tags": [] + }, + { + "id": 58, + "isActive": true, + "balance": "$2,876.10", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Mamie Fischer", + "gender": "female", + "email": "mamiefischer@chorizon.com", + "phone": "+1 (948) 545-3901", + "address": "599 Hunterfly Place, Haena, Georgia, 6005", + "about": "Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n", + "registered": "2019-05-27T05:07:10 -02:00", + "latitude": 70.915079, + "longitude": -48.813584, + "tags": [ + "bug", + "wontfix", + "wontfix", + "good first issue" + ] + }, + { + "id": 59, + "isActive": true, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ] + }, + { + "id": 60, + "isActive": true, + "balance": "$1,770.93", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "brown", + "name": "Jody Herrera", + "gender": "female", + "email": "jodyherrera@chorizon.com", + "phone": "+1 (890) 583-3222", + "address": "261 Jay Street, Strykersville, Ohio, 9248", + "about": "Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n", + "registered": "2016-05-21T01:00:02 -02:00", + "latitude": -36.846586, + "longitude": 131.156223, + "tags": [] + }, + { + "id": 61, + "isActive": false, + "balance": "$2,813.41", + "picture": "http://placehold.it/32x32", + "age": 37, + "color": "Green", + "name": "Charles Castillo", + "gender": "male", + "email": "charlescastillo@chorizon.com", + "phone": "+1 (934) 467-2108", + "address": "675 Morton Street, Rew, Pennsylvania, 137", + "about": "Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n", + "registered": "2019-06-10T02:54:22 -02:00", + "latitude": -16.423202, + "longitude": -146.293752, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 62, + "isActive": true, + "balance": "$3,341.35", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "blue", + "name": "Estelle Ramirez", + "gender": "female", + "email": "estelleramirez@chorizon.com", + "phone": "+1 (816) 459-2073", + "address": "636 Nolans Lane, Camptown, California, 7794", + "about": "Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n", + "registered": "2015-02-14T01:05:50 -01:00", + "latitude": -46.591249, + "longitude": -83.385587, + "tags": [ + "good first issue", + "bug" + ] + }, + { + "id": 63, + "isActive": true, + "balance": "$2,478.30", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "blue", + "name": "Knowles Hebert", + "gender": "male", + "email": "knowleshebert@chorizon.com", + "phone": "+1 (819) 409-2308", + "address": "361 Kathleen Court, Gratton, Connecticut, 7254", + "about": "Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n", + "registered": "2016-03-08T08:34:52 -01:00", + "latitude": 71.042482, + "longitude": 152.460406, + "tags": [ + "good first issue", + "wontfix" + ] + }, + { + "id": 64, + "isActive": false, + "balance": "$2,559.09", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Thelma Mckenzie", + "gender": "female", + "email": "thelmamckenzie@chorizon.com", + "phone": "+1 (941) 596-2777", + "address": "202 Leonard Street, Riverton, Illinois, 8577", + "about": "Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n", + "registered": "2020-04-14T12:43:06 -02:00", + "latitude": 16.026129, + "longitude": 105.464476, + "tags": [] + }, + { + "id": 65, + "isActive": true, + "balance": "$1,025.08", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Carole Rowland", + "gender": "female", + "email": "carolerowland@chorizon.com", + "phone": "+1 (862) 558-3448", + "address": "941 Melba Court, Bluetown, Florida, 9555", + "about": "Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n", + "registered": "2014-12-01T05:55:35 -01:00", + "latitude": -0.191998, + "longitude": 43.389652, + "tags": [ + "wontfix" + ] + }, + { + "id": 66, + "isActive": true, + "balance": "$1,061.49", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "brown", + "name": "Higgins Aguilar", + "gender": "male", + "email": "higginsaguilar@chorizon.com", + "phone": "+1 (911) 540-3791", + "address": "132 Sackman Street, Layhill, Guam, 8729", + "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", + "registered": "2015-04-05T02:10:07 -02:00", + "latitude": 74.702813, + "longitude": 151.314972, + "tags": [ + "bug" + ] + }, + { + "id": 67, + "isActive": true, + "balance": "$3,510.14", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Ilene Gillespie", + "gender": "female", + "email": "ilenegillespie@chorizon.com", + "phone": "+1 (937) 575-2676", + "address": "835 Lake Street, Naomi, Alabama, 4131", + "about": "Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n", + "registered": "2015-06-28T09:41:45 -02:00", + "latitude": 71.573342, + "longitude": -95.295989, + "tags": [ + "wontfix", + "wontfix" + ] + }, + { + "id": 68, + "isActive": false, + "balance": "$1,539.98", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Angelina Dyer", + "gender": "female", + "email": "angelinadyer@chorizon.com", + "phone": "+1 (948) 574-3949", + "address": "575 Division Place, Gorham, Louisiana, 3458", + "about": "Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n", + "registered": "2014-07-08T06:34:36 -02:00", + "latitude": -85.649593, + "longitude": 66.126018, + "tags": [ + "good first issue" + ] + }, + { + "id": 69, + "isActive": true, + "balance": "$3,367.69", + "picture": "http://placehold.it/32x32", + "age": 30, + "color": "brown", + "name": "Marks Burt", + "gender": "male", + "email": "marksburt@chorizon.com", + "phone": "+1 (895) 497-3138", + "address": "819 Village Road, Wadsworth, Delaware, 6099", + "about": "Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n", + "registered": "2014-08-31T06:12:18 -02:00", + "latitude": 26.854112, + "longitude": -143.313948, + "tags": [ + "good first issue" + ] + }, + { + "id": 70, + "isActive": false, + "balance": "$3,755.72", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "blue", + "name": "Glass Perkins", + "gender": "male", + "email": "glassperkins@chorizon.com", + "phone": "+1 (923) 486-3725", + "address": "899 Roosevelt Court, Belleview, Idaho, 1737", + "about": "Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n", + "registered": "2015-05-22T05:44:33 -02:00", + "latitude": 54.27147, + "longitude": -65.065604, + "tags": [ + "wontfix" + ] + }, + { + "id": 71, + "isActive": true, + "balance": "$3,381.63", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Candace Sawyer", + "gender": "female", + "email": "candacesawyer@chorizon.com", + "phone": "+1 (830) 404-2636", + "address": "334 Arkansas Drive, Bordelonville, Tennessee, 8449", + "about": "Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n", + "registered": "2014-04-04T08:45:00 -02:00", + "latitude": 6.484262, + "longitude": -37.054928, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 72, + "isActive": true, + "balance": "$1,640.98", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Hendricks Martinez", + "gender": "male", + "email": "hendricksmartinez@chorizon.com", + "phone": "+1 (857) 566-3245", + "address": "636 Agate Court, Newry, Utah, 3304", + "about": "Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n", + "registered": "2018-06-15T10:36:11 -02:00", + "latitude": 86.746034, + "longitude": 10.347893, + "tags": [ + "new issue" + ] + }, + { + "id": 73, + "isActive": false, + "balance": "$1,239.74", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "blue", + "name": "Eleanor Shepherd", + "gender": "female", + "email": "eleanorshepherd@chorizon.com", + "phone": "+1 (894) 567-2617", + "address": "670 Lafayette Walk, Darlington, Palau, 8803", + "about": "Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n", + "registered": "2020-02-29T12:15:28 -01:00", + "latitude": 35.749621, + "longitude": -94.40842, + "tags": [ + "good first issue", + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 74, + "isActive": true, + "balance": "$1,180.90", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Stark Wong", + "gender": "male", + "email": "starkwong@chorizon.com", + "phone": "+1 (805) 575-3055", + "address": "522 Bond Street, Bawcomville, Wisconsin, 324", + "about": "Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n", + "registered": "2020-01-25T10:47:48 -01:00", + "latitude": -80.452139, + "longitude": 160.72546, + "tags": [ + "wontfix" + ] + }, + { + "id": 75, + "isActive": false, + "balance": "$1,913.42", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Emma Jacobs", + "gender": "female", + "email": "emmajacobs@chorizon.com", + "phone": "+1 (899) 554-3847", + "address": "173 Tapscott Street, Esmont, Maine, 7450", + "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", + "registered": "2019-03-29T06:24:13 -01:00", + "latitude": -35.53722, + "longitude": 155.703874, + "tags": [] + }, + { + "id": 76, + "isActive": false, + "balance": "$1,274.29", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "Green", + "name": "Clarice Gardner", + "gender": "female", + "email": "claricegardner@chorizon.com", + "phone": "+1 (810) 407-3258", + "address": "894 Brooklyn Road, Utting, New Hampshire, 6404", + "about": "Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n", + "registered": "2014-10-20T10:13:32 -02:00", + "latitude": 17.11935, + "longitude": 65.38197, + "tags": [ + "new issue", + "wontfix" + ] + } +] diff --git a/tests/common/index.rs b/meilisearch-http/tests/common/index.rs similarity index 100% rename from tests/common/index.rs rename to meilisearch-http/tests/common/index.rs diff --git a/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs similarity index 100% rename from tests/common/mod.rs rename to meilisearch-http/tests/common/mod.rs diff --git a/tests/common/server.rs b/meilisearch-http/tests/common/server.rs similarity index 100% rename from tests/common/server.rs rename to meilisearch-http/tests/common/server.rs diff --git a/tests/common/service.rs b/meilisearch-http/tests/common/service.rs similarity index 100% rename from tests/common/service.rs rename to meilisearch-http/tests/common/service.rs diff --git a/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs similarity index 100% rename from tests/documents/add_documents.rs rename to meilisearch-http/tests/documents/add_documents.rs diff --git a/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs similarity index 100% rename from tests/documents/delete_documents.rs rename to meilisearch-http/tests/documents/delete_documents.rs diff --git a/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs similarity index 100% rename from tests/documents/get_documents.rs rename to meilisearch-http/tests/documents/get_documents.rs diff --git a/tests/documents/mod.rs b/meilisearch-http/tests/documents/mod.rs similarity index 100% rename from tests/documents/mod.rs rename to meilisearch-http/tests/documents/mod.rs diff --git a/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs similarity index 100% rename from tests/index/create_index.rs rename to meilisearch-http/tests/index/create_index.rs diff --git a/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs similarity index 100% rename from tests/index/delete_index.rs rename to meilisearch-http/tests/index/delete_index.rs diff --git a/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs similarity index 100% rename from tests/index/get_index.rs rename to meilisearch-http/tests/index/get_index.rs diff --git a/tests/index/mod.rs b/meilisearch-http/tests/index/mod.rs similarity index 100% rename from tests/index/mod.rs rename to meilisearch-http/tests/index/mod.rs diff --git a/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs similarity index 100% rename from tests/index/update_index.rs rename to meilisearch-http/tests/index/update_index.rs diff --git a/tests/integration.rs b/meilisearch-http/tests/integration.rs similarity index 100% rename from tests/integration.rs rename to meilisearch-http/tests/integration.rs diff --git a/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs similarity index 100% rename from tests/search/mod.rs rename to meilisearch-http/tests/search/mod.rs diff --git a/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs similarity index 100% rename from tests/settings/get_settings.rs rename to meilisearch-http/tests/settings/get_settings.rs diff --git a/tests/settings/mod.rs b/meilisearch-http/tests/settings/mod.rs similarity index 100% rename from tests/settings/mod.rs rename to meilisearch-http/tests/settings/mod.rs diff --git a/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs similarity index 100% rename from tests/updates/mod.rs rename to meilisearch-http/tests/updates/mod.rs From d1be3d60df8006c26ab10de6e91007513479542d Mon Sep 17 00:00:00 2001 From: mpostma Date: Sun, 28 Feb 2021 16:43:53 +0100 Subject: [PATCH 094/522] run tests on all pushed --- .github/workflows/rust.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3c13d1be2..ea4b41a6a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,10 +1,6 @@ name: Rust -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: push env: CARGO_TERM_COLOR: always From 4316d991a2b8db55a6a9b15a5e7baa620e87a6f7 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sun, 28 Feb 2021 16:56:05 +0100 Subject: [PATCH 095/522] add docker recipe --- .dockerignore | 5 +++++ Dockerfile | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..364510117 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +target +Dockerfile +.dockerignore +.git +.gitignore diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..9898d02db --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Compile +FROM alpine:3.10 AS compiler + +RUN apk update --quiet +RUN apk add curl +RUN apk add build-base + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +WORKDIR /meilisearch + +COPY . . + +ENV RUSTFLAGS="-C target-feature=-crt-static" + +RUN $HOME/.cargo/bin/cargo build --release + +# Run +FROM alpine:3.10 + +RUN apk add -q --no-cache libgcc tini + +COPY --from=compiler /meilisearch/target/release/meilisearch . + +ENV MEILI_HTTP_ADDR 0.0.0.0:7700 +EXPOSE 7700/tcp + +ENTRYPOINT ["tini", "--"] +CMD ./meilisearch From e1e5935e3c879056c01a17b1ce01d906c956c63d Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Mar 2021 14:39:57 +0100 Subject: [PATCH 096/522] CI recipes --- .github/workflows/create_artifacts.yml | 38 +++++++++++++++++++++++++ .github/workflows/publish_to_docker.yml | 19 +++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 .github/workflows/create_artifacts.yml create mode 100644 .github/workflows/publish_to_docker.yml diff --git a/.github/workflows/create_artifacts.yml b/.github/workflows/create_artifacts.yml new file mode 100644 index 000000000..bb4546c06 --- /dev/null +++ b/.github/workflows/create_artifacts.yml @@ -0,0 +1,38 @@ +name: Create artifacts + +on: + push: + tags: + - v*-alpha.* + +jobs: + nightly: + name: Build Nighlty ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + artifact_name: test-ci + asset_name: meilisearch-linux-amd64 + - os: macos-latest + artifact_name: test-ci + asset_name: meilisearch-macos-amd64 + - os: windows-latest + artifact_name: test-ci.exe + asset_name: meilisearch-windows-amd64.exe + steps: + - uses: hecrj/setup-rust-action@master + with: + rust-version: stable + - uses: actions/checkout@v1 + - name: Build + run: cargo build --release --locked + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v1-release + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: target/release/${{ matrix.artifact_name }} + asset_name: ${{ matrix.asset_name }} + tag: ${{ github.ref }} diff --git a/.github/workflows/publish_to_docker.yml b/.github/workflows/publish_to_docker.yml new file mode 100644 index 000000000..92a8bf5db --- /dev/null +++ b/.github/workflows/publish_to_docker.yml @@ -0,0 +1,19 @@ +name: Publish to dockerhub + +on: + push: + tags: + - v*-alpha.* + +jobs: + publish: + name: Publishing to dockerhub + runs-on: ubuntu-latest + steps: + - name: Publish to Registry + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: getmeili/meilisearch + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + tag_names: true From 9227b7cb2f0529475a2eccc40ffe97538cbf4e34 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Mar 2021 15:30:48 +0100 Subject: [PATCH 097/522] remove data.ms --- .../data.mdb | Bin 32768 -> 0 bytes .../lock.mdb | Bin 8192 -> 0 bytes .../data.mdb | Bin 45056 -> 0 bytes .../lock.mdb | Bin 8192 -> 0 bytes meilisearch-http/data.ms/updates/data.mdb | Bin 65536 -> 0 bytes meilisearch-http/data.ms/updates/lock.mdb | Bin 8192 -> 0 bytes 6 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/data.mdb delete mode 100644 meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/lock.mdb delete mode 100644 meilisearch-http/data.ms/indexes/index-eb9d0392-4386-4fa1-b68c-dac0d93bb16e/data.mdb delete mode 100644 meilisearch-http/data.ms/indexes/index-eb9d0392-4386-4fa1-b68c-dac0d93bb16e/lock.mdb delete mode 100644 meilisearch-http/data.ms/updates/data.mdb delete mode 100644 meilisearch-http/data.ms/updates/lock.mdb diff --git a/meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/data.mdb b/meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/data.mdb deleted file mode 100644 index e1c9db873b20be58dd26e5c90038cb00586ddddb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32768 zcmeI)O-{l<7{>9}`h_u27c2}KZ_vaAD{n&tlNz7~@N4G@IEyDREH7gZ!ASymUp; ztEjm>>bIHq-_A?5=bID=AbMa(RSkDNtw@n?-$v+JbSj4m2+FS_U@H*qH(+|C%%U3qn6mtL@ooAl#B zoOGu?Jr`@&on01i>Ja&z*sx9`?oN9f*Zl#3)ofU)heL9IZ$bqD1Q0*~0R#|0009IL zK;Ul#{Fnmy|M$RSBQdk({~P{$0;}46_#EDczKIF~2q1s}0tg_000IagfB*tZBjCmq z{>%Kodl_HJ^xizJ-`8{V+Ezd1_iESs^>n<-(Q1%GJIUt%S^d8>KWop#rd|&<4u72a=1fQU%;rajo diff --git a/meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/lock.mdb b/meilisearch-http/data.ms/indexes/index-7bcbd084-78eb-4510-88ef-a0859040bd84/lock.mdb deleted file mode 100644 index 7bae5680e543d283e07ca391af515f86818e93f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeIup$z~a3`Idutj|P*NmzjuFbL9sLtb5f^307sTTVH}t8v(Wx$I~C>Mogy009C7 k2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5cnjp0Eb)zb^rhX diff --git a/meilisearch-http/data.ms/indexes/index-eb9d0392-4386-4fa1-b68c-dac0d93bb16e/data.mdb b/meilisearch-http/data.ms/indexes/index-eb9d0392-4386-4fa1-b68c-dac0d93bb16e/data.mdb deleted file mode 100644 index 92d43bcc390866e63d79078d2ad9d880b50b004c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 45056 zcmeI*!B5jr9Ki9ni8vr@7sJ5_@z9GGnlNKD>AxX&Qdt)ZY+cqO5JFz(~X=l(yf1G4|Zh6(}*AbVFqU{F!*W}_^vI3lHJ6;Q<#@~cX&J9(jnW3o zj+gFOa+zC!rCF&t?I?Qg_nnj5#l@V?|9jSBo4fxD5-JEFfB*sr zAbCJis9BWPV@T$@==kpVQSKPUruu{&&U+U^4;; zAbl)~fo?ObDs;fa*rHY}+$ zxyzG!x9-2*H6P`{m80AbM?X$;IjIB)5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U KAV7e?AAuXq{|@{B diff --git a/meilisearch-http/data.ms/updates/data.mdb b/meilisearch-http/data.ms/updates/data.mdb deleted file mode 100644 index dea8bd465ebff560d5aa6cf37a8b63b65e23eb00..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65536 zcmeI*y>8o96u|LY^~0|1ibhcsL5G0gYy~9sVM$X0bZMueXqOa4K#A9}8i`UQ%821= z%iu9j5@hU96n%mWojUau3iJh9ejUWj~4k=l7rMj4ucvfB*srAb*~Bsv~6!V9(ZB2?dwU>b~Nq{yz$|qC;HH82h)Ckwqxnw=~Pd({~&JX+OAWv-HN;W zo!x3XyY0Gj&uKNAZmZhdeHiuF3w-_T`)My8P25(rGli72oUH zeyiH)I1L@`G6}tLsQquo!>*o8dcos2j1&!3Z#l_GdORMEqfvUnf!FW(W=zmF2i|DM z)Xzq`8+B=V=0rzFA9S_pg=W7OXwxyf!~S$&CBTL24@Xvf%%-zrTHRp~Mu^e32x!>x z#+EtzaYOR&as2;A;U}5S%}J+CmOO$(dyPFLfp#Pm%}aPtLNROuKQ_ytoEnUf3@${uliibRPLq6wqorv#6k7{wV%w zb@q~(@{M|{TCZ2Dc05!5I@KGzgn9gLS#K~KspT|Xw^n;-%{B`4xc>h^`GENU%CCL~ zKmY**5I_I{1Q0*~0R(Q3zfL6SEcxK7pCZPe^I{-1rE zUvwq)x!O1Ov-*Ge&#WMT00IagfB*srAbf7g)u|5-B@2q1s} z0tg_000IagfB*sr+ya4{^Zfs^_5bC5LLC1~{{IE<|GNd(Hm8UH0tg_000IagfB*sr zTxWrsTmN73@si({{QF{lznSo_D~Zb`zaNF<$w%_{lDC)blK+=}WjtvozsvaY-aqM& z_y1+UED%5d0R#|0009ILKmY**u7SYKjsH*Lz485j@?d~c#ag4hyA!!DAkP&L|DXI1v5o)&2q1s}0tg_000Iag@NNYz5dTX;J&&5b&Tt%R z|3uV${<+OZ?q!v~bl?KLtV#Ty4lhyu*`d3n`tvhcCjOT?0W$v}j{iSXFN|M|c=kUZ z1Q0*~0R#|0009ILKmY**5V%SKnYVvl-9C9vV6hIMjX&t|36ykJ+c4* diff --git a/meilisearch-http/data.ms/updates/lock.mdb b/meilisearch-http/data.ms/updates/lock.mdb deleted file mode 100644 index b5c5d308cf28870d5d05eb7ef611b55192e6668a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeIuu?>JA5CA}^UCL=(!xb#;+{8GErL7}q6i7&ji38jh|IhPK+`gS_3^6>B^4o?f zl_qz2Qt#IN*SqGUJh*a{`~FzwVJs(=009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs M0RjXF5cnf-1Ev!Xwg3PC From ac2af4354d9d2c55d87199e7ea9928d4c1d8c574 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Mar 2021 15:35:32 +0100 Subject: [PATCH 098/522] remove actor index controller --- .../actor_index_controller/update_handler.rs | 260 ----------- .../actor_index_controller/update_store.rs | 423 ------------------ 2 files changed, 683 deletions(-) delete mode 100644 meilisearch-http/src/index_controller/actor_index_controller/update_handler.rs delete mode 100644 meilisearch-http/src/index_controller/actor_index_controller/update_store.rs diff --git a/meilisearch-http/src/index_controller/actor_index_controller/update_handler.rs b/meilisearch-http/src/index_controller/actor_index_controller/update_handler.rs deleted file mode 100644 index d9ac2f866..000000000 --- a/meilisearch-http/src/index_controller/actor_index_controller/update_handler.rs +++ /dev/null @@ -1,260 +0,0 @@ -use std::collections::HashMap; -use std::io; -use std::fs::File; - -use anyhow::Result; -use flate2::read::GzDecoder; -use grenad::CompressionType; -use log::info; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use milli::Index; -use rayon::ThreadPool; - -use crate::index_controller::updates::{Failed, Processed, Processing}; -use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; -use crate::option::IndexerOpts; - -pub struct UpdateHandler { - max_nb_chunks: Option, - chunk_compression_level: Option, - thread_pool: ThreadPool, - log_frequency: usize, - max_memory: usize, - linked_hash_map_size: usize, - chunk_compression_type: CompressionType, - chunk_fusing_shrink_size: u64, -} - -impl UpdateHandler { - pub fn new( - opt: &IndexerOpts, - ) -> anyhow::Result { - let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(opt.indexing_jobs.unwrap_or(0)) - .build()?; - Ok(Self { - max_nb_chunks: opt.max_nb_chunks, - chunk_compression_level: opt.chunk_compression_level, - thread_pool, - log_frequency: opt.log_every_n, - max_memory: opt.max_memory.get_bytes() as usize, - linked_hash_map_size: opt.linked_hash_map_size, - chunk_compression_type: opt.chunk_compression_type, - chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), - }) - } - - fn update_buidler(&self, update_id: u64) -> UpdateBuilder { - // We prepare the update by using the update builder. - let mut update_builder = UpdateBuilder::new(update_id); - if let Some(max_nb_chunks) = self.max_nb_chunks { - update_builder.max_nb_chunks(max_nb_chunks); - } - if let Some(chunk_compression_level) = self.chunk_compression_level { - update_builder.chunk_compression_level(chunk_compression_level); - } - update_builder.thread_pool(&self.thread_pool); - update_builder.log_every_n(self.log_frequency); - update_builder.max_memory(self.max_memory); - update_builder.linked_hash_map_size(self.linked_hash_map_size); - update_builder.chunk_compression_type(self.chunk_compression_type); - update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); - update_builder - } - - fn update_documents( - &self, - format: UpdateFormat, - method: IndexDocumentsMethod, - content: File, - update_builder: UpdateBuilder, - primary_key: Option<&str>, - index: &Index, - ) -> anyhow::Result { - info!("performing document addition"); - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - - // Set the primary key if not set already, ignore if already set. - match (index.primary_key(&wtxn)?, primary_key) { - (None, Some(ref primary_key)) => { - index.put_primary_key(&mut wtxn, primary_key)?; - } - _ => (), - } - - let mut builder = update_builder.index_documents(&mut wtxn, index); - builder.update_format(format); - builder.index_documents_method(method); - - let gzipped = false; - let reader = if gzipped { - Box::new(GzDecoder::new(content)) - } else { - Box::new(content) as Box - }; - - let result = builder.execute(reader, |indexing_step, update_id| { - info!("update {}: {:?}", update_id, indexing_step) - }); - - info!("document addition done: {:?}", result); - - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn clear_documents(&self, update_builder: UpdateBuilder, index: &Index) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let builder = update_builder.clear_documents(&mut wtxn, index); - - match builder.execute() { - Ok(_count) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_settings( - &self, - settings: &Settings, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let mut builder = update_builder.settings(&mut wtxn, index); - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.searchable_attributes { - match names { - Some(names) => builder.set_searchable_fields(names.clone()), - None => builder.reset_searchable_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.displayed_attributes { - match names { - Some(names) => builder.set_displayed_fields(names.clone()), - None => builder.reset_displayed_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); - builder.set_faceted_fields(facet_types); - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { - match criteria { - Some(criteria) => builder.set_criteria(criteria.clone()), - None => builder.reset_criteria(), - } - } - - let result = builder - .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_facets( - &self, - levels: &Facets, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let mut builder = update_builder.facets(&mut wtxn, index); - if let Some(value) = levels.level_group_size { - builder.level_group_size(value); - } - if let Some(value) = levels.min_level_size { - builder.min_level_size(value); - } - match builder.execute() { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn delete_documents( - &self, - document_ids: File, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - let ids: Vec = serde_json::from_reader(document_ids)?; - let mut txn = index.write_txn()?; - let mut builder = update_builder.delete_documents(&mut txn, index)?; - - // We ignore unexisting document ids - ids.iter().for_each(|id| { builder.delete_external_id(id); }); - - match builder.execute() { - Ok(deleted) => txn - .commit() - .and(Ok(UpdateResult::DocumentDeletion { deleted })) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } - - pub fn handle_update( - &self, - meta: Processing, - content: File, - index: &Index, - ) -> Result, Failed> { - use UpdateMeta::*; - - let update_id = meta.id(); - - let update_builder = self.update_buidler(update_id); - - let result = match meta.meta() { - DocumentsAddition { - method, - format, - primary_key, - } => self.update_documents( - *format, - *method, - content, - update_builder, - primary_key.as_deref(), - index, - ), - ClearDocuments => self.clear_documents(update_builder, index), - DeleteDocuments => self.delete_documents(content, update_builder, index), - Settings(settings) => self.update_settings(settings, update_builder, index), - Facets(levels) => self.update_facets(levels, update_builder, index), - }; - - match result { - Ok(result) => Ok(meta.process(result)), - Err(e) => Err(meta.fail(e.to_string())), - } - } -} diff --git a/meilisearch-http/src/index_controller/actor_index_controller/update_store.rs b/meilisearch-http/src/index_controller/actor_index_controller/update_store.rs deleted file mode 100644 index 371ac7bd9..000000000 --- a/meilisearch-http/src/index_controller/actor_index_controller/update_store.rs +++ /dev/null @@ -1,423 +0,0 @@ -use std::path::Path; -use std::sync::{Arc, RwLock}; -use std::io::{Cursor, SeekFrom, Seek}; - -use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; -use heed::{EnvOpenOptions, Env, Database}; -use serde::{Serialize, Deserialize}; -use std::fs::File; -use uuid::Uuid; - -use crate::index_controller::updates::*; - -type BEU64 = heed::zerocopy::U64; - -#[derive(Clone)] -pub struct UpdateStore { - env: Env, - pending_meta: Database, SerdeJson>>, - pending: Database, ByteSlice>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, - processing: Arc>>>, - notification_sender: Sender<()>, -} - -pub trait HandleUpdate { - fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed>; -} - -impl HandleUpdate for F -where F: FnMut(Processing, File) -> Result, Failed> -{ - fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed> { - self(meta, content) - } -} - -impl UpdateStore -where - M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, - N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, - E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, -{ - pub fn open( - mut options: EnvOpenOptions, - path: P, - mut update_handler: U, - ) -> heed::Result> - where - P: AsRef, - U: HandleUpdate + Send + 'static, - { - options.max_dbs(5); - - let env = options.open(path)?; - let pending_meta = env.create_database(Some("pending-meta"))?; - let pending = env.create_database(Some("pending"))?; - let processed_meta = env.create_database(Some("processed-meta"))?; - let aborted_meta = env.create_database(Some("aborted-meta"))?; - let failed_meta = env.create_database(Some("failed-meta"))?; - let processing = Arc::new(RwLock::new(None)); - - let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); - // Send a first notification to trigger the process. - let _ = notification_sender.send(()); - - let update_store = Arc::new(UpdateStore { - env, - pending, - pending_meta, - processed_meta, - aborted_meta, - notification_sender, - failed_meta, - processing, - }); - - // We need a weak reference so we can take ownership on the arc later when we - // want to close the index. - let update_store_weak = Arc::downgrade(&update_store); - std::thread::spawn(move || { - // Block and wait for something to process. - 'outer: for _ in notification_receiver { - loop { - match update_store_weak.upgrade() { - Some(update_store) => { - match update_store.process_pending_update(&mut update_handler) { - Ok(Some(_)) => (), - Ok(None) => break, - Err(e) => eprintln!("error while processing update: {}", e), - } - } - // the ownership on the arc has been taken, we need to exit. - None => break 'outer, - } - } - } - }); - - Ok(update_store) - } - - pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { - self.env.prepare_for_closing() - } - - /// Returns the new biggest id to use to store the new update. - fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { - let last_pending = self.pending_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_processed = self.processed_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_aborted = self.aborted_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_update_id = [last_pending, last_processed, last_aborted] - .iter() - .copied() - .flatten() - .max(); - - match last_update_id { - Some(last_id) => Ok(last_id + 1), - None => Ok(0), - } - } - - /// Registers the update content in the pending store and the meta - /// into the pending-meta store. Returns the new unique update id. - pub fn register_update( - &self, - meta: M, - content: &[u8], - index_uuid: Uuid, - ) -> heed::Result> { - let mut wtxn = self.env.write_txn()?; - - // We ask the update store to give us a new update id, this is safe, - // no other update can have the same id because we use a write txn before - // asking for the id and registering it so other update registering - // will be forced to wait for a new write txn. - let update_id = self.new_update_id(&wtxn)?; - let update_key = BEU64::new(update_id); - - let meta = Pending::new(meta, update_id, index_uuid); - self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, content)?; - - wtxn.commit()?; - - if let Err(e) = self.notification_sender.try_send(()) { - assert!(!e.is_disconnected(), "update notification channel is disconnected"); - } - Ok(meta) - } - /// Executes the user provided function on the next pending update (the one with the lowest id). - /// This is asynchronous as it let the user process the update with a read-only txn and - /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, handler: &mut U) -> heed::Result> - where - U: HandleUpdate + Send + 'static, - { - // Create a read transaction to be able to retrieve the pending update in order. - let rtxn = self.env.read_txn()?; - let first_meta = self.pending_meta.first(&rtxn)?; - - // If there is a pending update we process and only keep - // a reader while processing it, not a writer. - match first_meta { - Some((first_id, pending)) => { - let first_content = self.pending - .get(&rtxn, &first_id)? - .expect("associated update content"); - - // we change the state of the update from pending to processing before we pass it - // to the update handler. Processing store is non persistent to be able recover - // from a failure - let processing = pending.processing(); - self.processing - .write() - .unwrap() - .replace(processing.clone()); - let mut cursor = Cursor::new(first_content); - let mut file = tempfile::tempfile()?; - std::io::copy(&mut cursor, &mut file)?; - file.seek(SeekFrom::Start(0))?; - // Process the pending update using the provided user function. - let result = handler.handle_update(processing, file); - drop(rtxn); - - // Once the pending update have been successfully processed - // we must remove the content from the pending and processing stores and - // write the *new* meta to the processed-meta store and commit. - let mut wtxn = self.env.write_txn()?; - self.processing - .write() - .unwrap() - .take(); - self.pending_meta.delete(&mut wtxn, &first_id)?; - self.pending.delete(&mut wtxn, &first_id)?; - match result { - Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, - Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, - } - wtxn.commit()?; - - Ok(Some(())) - }, - None => Ok(None) - } - } - - /// Execute the user defined function with the meta-store iterators, the first - /// iterator is the *processed* meta one, the second the *aborted* meta one - /// and, the last is the *pending* meta one. - pub fn iter_metas(&self, mut f: F) -> heed::Result - where - F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - ) -> heed::Result, - { - let rtxn = self.env.read_txn()?; - - // We get the pending, processed and aborted meta iterators. - let processed_iter = self.processed_meta.iter(&rtxn)?; - let aborted_iter = self.aborted_meta.iter(&rtxn)?; - let pending_iter = self.pending_meta.iter(&rtxn)?; - let processing = self.processing.read().unwrap().clone(); - let failed_iter = self.failed_meta.iter(&rtxn)?; - - // We execute the user defined function with both iterators. - (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) - } - - /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta(&self, update_id: u64) -> heed::Result>> { - let rtxn = self.env.read_txn()?; - let key = BEU64::new(update_id); - - if let Some(ref meta) = *self.processing.read().unwrap() { - if meta.id() == update_id { - return Ok(Some(UpdateStatus::Processing(meta.clone()))); - } - } - - if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Pending(meta))); - } - - if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Processed(meta))); - } - - if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Aborted(meta))); - } - - if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Failed(meta))); - } - - Ok(None) - } - - /// Aborts an update, an aborted update content is deleted and - /// the meta of it is moved into the aborted updates database. - /// - /// Trying to abort an update that is currently being processed, an update - /// that as already been processed or which doesn't actually exist, will - /// return `None`. - #[allow(dead_code)] - pub fn abort_update(&self, update_id: u64) -> heed::Result>> { - let mut wtxn = self.env.write_txn()?; - let key = BEU64::new(update_id); - - // We cannot abort an update that is currently being processed. - if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { - return Ok(None); - } - - let pending = match self.pending_meta.get(&wtxn, &key)? { - Some(meta) => meta, - None => return Ok(None), - }; - - let aborted = pending.abort(); - - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - - wtxn.commit()?; - - Ok(Some(aborted)) - } - - /// Aborts all the pending updates, and not the one being currently processed. - /// Returns the update metas and ids that were successfully aborted. - #[allow(dead_code)] - pub fn abort_pendings(&self) -> heed::Result)>> { - let mut wtxn = self.env.write_txn()?; - let mut aborted_updates = Vec::new(); - - // We skip the first pending update as it is currently being processed. - for result in self.pending_meta.iter(&wtxn)?.skip(1) { - let (key, pending) = result?; - let id = key.get(); - aborted_updates.push((id, pending.abort())); - } - - for (id, aborted) in &aborted_updates { - let key = BEU64::new(*id); - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - } - - wtxn.commit()?; - - Ok(aborted_updates) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::thread; - use std::time::{Duration, Instant}; - - impl HandleUpdate for F - where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { - fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { - self(meta, content) - } - } - - #[test] - fn simple() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let meta = String::from("kiki"); - let update = update_store.register_update(meta, &[]).unwrap(); - thread::sleep(Duration::from_millis(100)); - let meta = update_store.meta(update.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - } - - #[test] - #[ignore] - fn long_running_update() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let before_register = Instant::now(); - - let meta = String::from("kiki"); - let update_kiki = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("coco"); - let update_coco = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("cucu"); - let update_cucu = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - thread::sleep(Duration::from_millis(400 * 3 + 100)); - - let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "coco processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "cucu processed"); - } else { - panic!() - } - } -} From c4dfd5f0c3707e968cd2192e33d4ae42f4b6ca0b Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 27 Feb 2021 10:19:05 +0100 Subject: [PATCH 099/522] implement search and fix document addition --- src/data/mod.rs | 2 +- src/data/search.rs | 34 +- .../actor_index_controller/index_actor.rs | 128 +++++- .../actor_index_controller/mod.rs | 7 + .../actor_index_controller/update_handler.rs | 260 +++++++++++ .../actor_index_controller/update_store.rs | 423 ++++++++++++++++++ .../actor_index_controller/uuid_resolver.rs | 13 +- src/index_controller/mod.rs | 4 + src/routes/search.rs | 4 +- 9 files changed, 842 insertions(+), 33 deletions(-) create mode 100644 src/index_controller/actor_index_controller/update_handler.rs create mode 100644 src/index_controller/actor_index_controller/update_store.rs diff --git a/src/data/mod.rs b/src/data/mod.rs index 3d6858763..ecc8a7e2e 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -1,4 +1,4 @@ -mod search; +pub mod search; mod updates; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; diff --git a/src/data/search.rs b/src/data/search.rs index 23c6b463e..6ab792073 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -123,27 +123,27 @@ impl SearchQuery { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct SearchResult { - hits: Vec>, - nb_hits: u64, - query: String, - limit: usize, - offset: usize, - processing_time_ms: u128, + pub hits: Vec>, + pub nb_hits: u64, + pub query: String, + pub limit: usize, + pub offset: usize, + pub processing_time_ms: u128, #[serde(skip_serializing_if = "Option::is_none")] - facet_distributions: Option>>, + pub facet_distributions: Option>>, } -struct Highlighter<'a, A> { +pub struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, } impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { - fn new(stop_words: &'a fst::Set) -> Self { + pub fn new(stop_words: &'a fst::Set) -> Self { let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); Self { analyzer } } - fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { match value { Value::Null => Value::Null, Value::Bool(boolean) => Value::Bool(boolean), @@ -182,7 +182,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } } - fn highlight_record( + pub fn highlight_record( &self, object: &mut Map, words_to_highlight: &HashSet, @@ -199,16 +199,12 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } impl Data { - pub fn search>( + pub async fn search>( &self, - _index: S, - _search_query: SearchQuery, + index: S, + search_query: SearchQuery, ) -> anyhow::Result { - todo!() - //match self.index_controller.index(&index)? { - //Some(index) => Ok(search_query.perform(index)?), - //None => bail!("index {:?} doesn't exists", index.as_ref()), - //} + self.index_controller.search(index.as_ref().to_string(), search_query).await } pub async fn retrieve_documents( diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 6123ca774..0cb057a9b 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -1,6 +1,7 @@ use std::fs::{File, create_dir_all}; use std::path::{PathBuf, Path}; use std::sync::Arc; +use std::time::Instant; use chrono::Utc; use heed::EnvOpenOptions; @@ -11,8 +12,12 @@ use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use log::info; +use crate::data::SearchQuery; +use futures::stream::{StreamExt, Stream}; use super::update_handler::UpdateHandler; +use async_stream::stream; +use crate::data::SearchResult; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; use crate::option::IndexerOpts; @@ -22,7 +27,8 @@ type UpdateResult = std::result::Result, Failed, ret: oneshot::Sender> }, - Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, + Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, + Search { uuid: Uuid, query: SearchQuery, ret: oneshot::Sender> }, } struct IndexActor { @@ -43,6 +49,7 @@ pub enum IndexError { trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get_or_create(&self, uuid: Uuid) -> Result>; + async fn get(&self, uuid: Uuid) -> Result>>; } impl IndexActor { @@ -54,13 +61,109 @@ impl IndexActor { } async fn run(mut self) { - loop { - match self.inbox.recv().await { - Some(IndexMsg::CreateIndex { uuid, primary_key, ret }) => self.handle_create_index(uuid, primary_key, ret).await, - Some(IndexMsg::Update { ret, meta, data }) => self.handle_update(meta, data, ret).await, - None => break, + let stream = stream! { + loop { + match self.inbox.recv().await { + Some(msg) => yield msg, + None => break, + } } - } + }; + + stream.for_each_concurent(Some(10), |msg| { + match msg { + IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret), + IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret), + IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret), + } + }) + } + + async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { + let index = self.store.get(uuid).await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || { + + let before_search = Instant::now(); + let rtxn = index.read_txn().unwrap(); + + let mut search = index.search(&rtxn); + + if let Some(ref query) = query.q { + search.query(query); + } + + search.limit(query.limit); + search.offset(query.offset.unwrap_or_default()); + + //if let Some(ref facets) = query.facet_filters { + //if let Some(facets) = parse_facets(facets, index, &rtxn)? { + //search.facet_condition(facets); + //} + //} + let milli::SearchResult { + documents_ids, + found_words, + candidates, + .. + } = search.execute().unwrap(); + let mut documents = Vec::new(); + let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields_ids = index.displayed_fields_ids(&rtxn).unwrap(); + + let attributes_to_retrieve_ids = match query.attributes_to_retrieve { + Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, + Some(ref attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>() + .into(), + None => None, + }; + + let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { + (_, Some(ids)) => ids, + (Some(ids), None) => ids, + (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let stop_words = fst::Set::default(); + let highlighter = crate::data::search::Highlighter::new(&stop_words); + + for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { + let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); + if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { + highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + } + documents.push(object); + } + + let nb_hits = candidates.len(); + + let facet_distributions = match query.facet_distributions { + Some(ref fields) => { + let mut facet_distribution = index.facets_distribution(&rtxn); + if fields.iter().all(|f| f != "*") { + facet_distribution.facets(fields); + } + Some(facet_distribution.candidates(candidates).execute().unwrap()) + } + None => None, + }; + + let result = Ok(SearchResult { + hits: documents, + nb_hits, + query: query.q.clone().unwrap_or_default(), + limit: query.limit, + offset: query.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + facet_distributions, + }); + + ret.send(result) + }); + } async fn handle_create_index(&self, uuid: Uuid, primary_key: Option, ret: oneshot::Sender>) { @@ -107,6 +210,13 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } + + pub async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Search { uuid, query, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { @@ -162,6 +272,10 @@ impl IndexStore for MapIndexStore { Entry::Occupied(entry) => Ok(entry.get().clone()), } } + + async fn get(&self, uuid: Uuid) -> Result>> { + Ok(self.index_store.read().await.get(&uuid).cloned()) + } } impl MapIndexStore { diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 2936c59ea..646d9cf45 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -10,6 +10,7 @@ use uuid::Uuid; use super::IndexMetadata; use tokio::fs::File; use super::UpdateMeta; +use crate::data::{SearchResult, SearchQuery}; pub struct ActorIndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, @@ -97,4 +98,10 @@ impl IndexController for ActorIndexController { fn update_index(&self, name: String, index_settings: super::IndexSettings) -> anyhow::Result { todo!() } + + async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); + let result = self.index_handle.search(uuid, query).await?; + Ok(result) + } } diff --git a/src/index_controller/actor_index_controller/update_handler.rs b/src/index_controller/actor_index_controller/update_handler.rs new file mode 100644 index 000000000..d9ac2f866 --- /dev/null +++ b/src/index_controller/actor_index_controller/update_handler.rs @@ -0,0 +1,260 @@ +use std::collections::HashMap; +use std::io; +use std::fs::File; + +use anyhow::Result; +use flate2::read::GzDecoder; +use grenad::CompressionType; +use log::info; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::Index; +use rayon::ThreadPool; + +use crate::index_controller::updates::{Failed, Processed, Processing}; +use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; +use crate::option::IndexerOpts; + +pub struct UpdateHandler { + max_nb_chunks: Option, + chunk_compression_level: Option, + thread_pool: ThreadPool, + log_frequency: usize, + max_memory: usize, + linked_hash_map_size: usize, + chunk_compression_type: CompressionType, + chunk_fusing_shrink_size: u64, +} + +impl UpdateHandler { + pub fn new( + opt: &IndexerOpts, + ) -> anyhow::Result { + let thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(opt.indexing_jobs.unwrap_or(0)) + .build()?; + Ok(Self { + max_nb_chunks: opt.max_nb_chunks, + chunk_compression_level: opt.chunk_compression_level, + thread_pool, + log_frequency: opt.log_every_n, + max_memory: opt.max_memory.get_bytes() as usize, + linked_hash_map_size: opt.linked_hash_map_size, + chunk_compression_type: opt.chunk_compression_type, + chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), + }) + } + + fn update_buidler(&self, update_id: u64) -> UpdateBuilder { + // We prepare the update by using the update builder. + let mut update_builder = UpdateBuilder::new(update_id); + if let Some(max_nb_chunks) = self.max_nb_chunks { + update_builder.max_nb_chunks(max_nb_chunks); + } + if let Some(chunk_compression_level) = self.chunk_compression_level { + update_builder.chunk_compression_level(chunk_compression_level); + } + update_builder.thread_pool(&self.thread_pool); + update_builder.log_every_n(self.log_frequency); + update_builder.max_memory(self.max_memory); + update_builder.linked_hash_map_size(self.linked_hash_map_size); + update_builder.chunk_compression_type(self.chunk_compression_type); + update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); + update_builder + } + + fn update_documents( + &self, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: File, + update_builder: UpdateBuilder, + primary_key: Option<&str>, + index: &Index, + ) -> anyhow::Result { + info!("performing document addition"); + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + + // Set the primary key if not set already, ignore if already set. + match (index.primary_key(&wtxn)?, primary_key) { + (None, Some(ref primary_key)) => { + index.put_primary_key(&mut wtxn, primary_key)?; + } + _ => (), + } + + let mut builder = update_builder.index_documents(&mut wtxn, index); + builder.update_format(format); + builder.index_documents_method(method); + + let gzipped = false; + let reader = if gzipped { + Box::new(GzDecoder::new(content)) + } else { + Box::new(content) as Box + }; + + let result = builder.execute(reader, |indexing_step, update_id| { + info!("update {}: {:?}", update_id, indexing_step) + }); + + info!("document addition done: {:?}", result); + + match result { + Ok(addition_result) => wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn clear_documents(&self, update_builder: UpdateBuilder, index: &Index) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, index); + + match builder.execute() { + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn update_settings( + &self, + settings: &Settings, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let mut builder = update_builder.settings(&mut wtxn, index); + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names.clone()), + None => builder.reset_searchable_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names.clone()), + None => builder.reset_displayed_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref facet_types) = settings.faceted_attributes { + let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + builder.set_faceted_fields(facet_types); + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref criteria) = settings.criteria { + match criteria { + Some(criteria) => builder.set_criteria(criteria.clone()), + None => builder.reset_criteria(), + } + } + + let result = builder + .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn update_facets( + &self, + levels: &Facets, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let mut builder = update_builder.facets(&mut wtxn, index); + if let Some(value) = levels.level_group_size { + builder.level_group_size(value); + } + if let Some(value) = levels.min_level_size { + builder.min_level_size(value); + } + match builder.execute() { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn delete_documents( + &self, + document_ids: File, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + let ids: Vec = serde_json::from_reader(document_ids)?; + let mut txn = index.write_txn()?; + let mut builder = update_builder.delete_documents(&mut txn, index)?; + + // We ignore unexisting document ids + ids.iter().for_each(|id| { builder.delete_external_id(id); }); + + match builder.execute() { + Ok(deleted) => txn + .commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + pub fn handle_update( + &self, + meta: Processing, + content: File, + index: &Index, + ) -> Result, Failed> { + use UpdateMeta::*; + + let update_id = meta.id(); + + let update_builder = self.update_buidler(update_id); + + let result = match meta.meta() { + DocumentsAddition { + method, + format, + primary_key, + } => self.update_documents( + *format, + *method, + content, + update_builder, + primary_key.as_deref(), + index, + ), + ClearDocuments => self.clear_documents(update_builder, index), + DeleteDocuments => self.delete_documents(content, update_builder, index), + Settings(settings) => self.update_settings(settings, update_builder, index), + Facets(levels) => self.update_facets(levels, update_builder, index), + }; + + match result { + Ok(result) => Ok(meta.process(result)), + Err(e) => Err(meta.fail(e.to_string())), + } + } +} diff --git a/src/index_controller/actor_index_controller/update_store.rs b/src/index_controller/actor_index_controller/update_store.rs new file mode 100644 index 000000000..371ac7bd9 --- /dev/null +++ b/src/index_controller/actor_index_controller/update_store.rs @@ -0,0 +1,423 @@ +use std::path::Path; +use std::sync::{Arc, RwLock}; +use std::io::{Cursor, SeekFrom, Seek}; + +use crossbeam_channel::Sender; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::{EnvOpenOptions, Env, Database}; +use serde::{Serialize, Deserialize}; +use std::fs::File; +use uuid::Uuid; + +use crate::index_controller::updates::*; + +type BEU64 = heed::zerocopy::U64; + +#[derive(Clone)] +pub struct UpdateStore { + env: Env, + pending_meta: Database, SerdeJson>>, + pending: Database, ByteSlice>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, + notification_sender: Sender<()>, +} + +pub trait HandleUpdate { + fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed>; +} + +impl HandleUpdate for F +where F: FnMut(Processing, File) -> Result, Failed> +{ + fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed> { + self(meta, content) + } +} + +impl UpdateStore +where + M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, + N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, + E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, +{ + pub fn open( + mut options: EnvOpenOptions, + path: P, + mut update_handler: U, + ) -> heed::Result> + where + P: AsRef, + U: HandleUpdate + Send + 'static, + { + options.max_dbs(5); + + let env = options.open(path)?; + let pending_meta = env.create_database(Some("pending-meta"))?; + let pending = env.create_database(Some("pending"))?; + let processed_meta = env.create_database(Some("processed-meta"))?; + let aborted_meta = env.create_database(Some("aborted-meta"))?; + let failed_meta = env.create_database(Some("failed-meta"))?; + let processing = Arc::new(RwLock::new(None)); + + let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + // Send a first notification to trigger the process. + let _ = notification_sender.send(()); + + let update_store = Arc::new(UpdateStore { + env, + pending, + pending_meta, + processed_meta, + aborted_meta, + notification_sender, + failed_meta, + processing, + }); + + // We need a weak reference so we can take ownership on the arc later when we + // want to close the index. + let update_store_weak = Arc::downgrade(&update_store); + std::thread::spawn(move || { + // Block and wait for something to process. + 'outer: for _ in notification_receiver { + loop { + match update_store_weak.upgrade() { + Some(update_store) => { + match update_store.process_pending_update(&mut update_handler) { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => eprintln!("error while processing update: {}", e), + } + } + // the ownership on the arc has been taken, we need to exit. + None => break 'outer, + } + } + } + }); + + Ok(update_store) + } + + pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { + self.env.prepare_for_closing() + } + + /// Returns the new biggest id to use to store the new update. + fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { + let last_pending = self.pending_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_processed = self.processed_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_aborted = self.aborted_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_update_id = [last_pending, last_processed, last_aborted] + .iter() + .copied() + .flatten() + .max(); + + match last_update_id { + Some(last_id) => Ok(last_id + 1), + None => Ok(0), + } + } + + /// Registers the update content in the pending store and the meta + /// into the pending-meta store. Returns the new unique update id. + pub fn register_update( + &self, + meta: M, + content: &[u8], + index_uuid: Uuid, + ) -> heed::Result> { + let mut wtxn = self.env.write_txn()?; + + // We ask the update store to give us a new update id, this is safe, + // no other update can have the same id because we use a write txn before + // asking for the id and registering it so other update registering + // will be forced to wait for a new write txn. + let update_id = self.new_update_id(&wtxn)?; + let update_key = BEU64::new(update_id); + + let meta = Pending::new(meta, update_id, index_uuid); + self.pending_meta.put(&mut wtxn, &update_key, &meta)?; + self.pending.put(&mut wtxn, &update_key, content)?; + + wtxn.commit()?; + + if let Err(e) = self.notification_sender.try_send(()) { + assert!(!e.is_disconnected(), "update notification channel is disconnected"); + } + Ok(meta) + } + /// Executes the user provided function on the next pending update (the one with the lowest id). + /// This is asynchronous as it let the user process the update with a read-only txn and + /// only writing the result meta to the processed-meta store *after* it has been processed. + fn process_pending_update(&self, handler: &mut U) -> heed::Result> + where + U: HandleUpdate + Send + 'static, + { + // Create a read transaction to be able to retrieve the pending update in order. + let rtxn = self.env.read_txn()?; + let first_meta = self.pending_meta.first(&rtxn)?; + + // If there is a pending update we process and only keep + // a reader while processing it, not a writer. + match first_meta { + Some((first_id, pending)) => { + let first_content = self.pending + .get(&rtxn, &first_id)? + .expect("associated update content"); + + // we change the state of the update from pending to processing before we pass it + // to the update handler. Processing store is non persistent to be able recover + // from a failure + let processing = pending.processing(); + self.processing + .write() + .unwrap() + .replace(processing.clone()); + let mut cursor = Cursor::new(first_content); + let mut file = tempfile::tempfile()?; + std::io::copy(&mut cursor, &mut file)?; + file.seek(SeekFrom::Start(0))?; + // Process the pending update using the provided user function. + let result = handler.handle_update(processing, file); + drop(rtxn); + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.processing + .write() + .unwrap() + .take(); + self.pending_meta.delete(&mut wtxn, &first_id)?; + self.pending.delete(&mut wtxn, &first_id)?; + match result { + Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, + Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, + } + wtxn.commit()?; + + Ok(Some(())) + }, + None => Ok(None) + } + } + + /// Execute the user defined function with the meta-store iterators, the first + /// iterator is the *processed* meta one, the second the *aborted* meta one + /// and, the last is the *pending* meta one. + pub fn iter_metas(&self, mut f: F) -> heed::Result + where + F: for<'a> FnMut( + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + ) -> heed::Result, + { + let rtxn = self.env.read_txn()?; + + // We get the pending, processed and aborted meta iterators. + let processed_iter = self.processed_meta.iter(&rtxn)?; + let aborted_iter = self.aborted_meta.iter(&rtxn)?; + let pending_iter = self.pending_meta.iter(&rtxn)?; + let processing = self.processing.read().unwrap().clone(); + let failed_iter = self.failed_meta.iter(&rtxn)?; + + // We execute the user defined function with both iterators. + (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + } + + /// Returns the update associated meta or `None` if the update doesn't exist. + pub fn meta(&self, update_id: u64) -> heed::Result>> { + let rtxn = self.env.read_txn()?; + let key = BEU64::new(update_id); + + if let Some(ref meta) = *self.processing.read().unwrap() { + if meta.id() == update_id { + return Ok(Some(UpdateStatus::Processing(meta.clone()))); + } + } + + if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Pending(meta))); + } + + if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Processed(meta))); + } + + if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Aborted(meta))); + } + + if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Failed(meta))); + } + + Ok(None) + } + + /// Aborts an update, an aborted update content is deleted and + /// the meta of it is moved into the aborted updates database. + /// + /// Trying to abort an update that is currently being processed, an update + /// that as already been processed or which doesn't actually exist, will + /// return `None`. + #[allow(dead_code)] + pub fn abort_update(&self, update_id: u64) -> heed::Result>> { + let mut wtxn = self.env.write_txn()?; + let key = BEU64::new(update_id); + + // We cannot abort an update that is currently being processed. + if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { + return Ok(None); + } + + let pending = match self.pending_meta.get(&wtxn, &key)? { + Some(meta) => meta, + None => return Ok(None), + }; + + let aborted = pending.abort(); + + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + + wtxn.commit()?; + + Ok(Some(aborted)) + } + + /// Aborts all the pending updates, and not the one being currently processed. + /// Returns the update metas and ids that were successfully aborted. + #[allow(dead_code)] + pub fn abort_pendings(&self) -> heed::Result)>> { + let mut wtxn = self.env.write_txn()?; + let mut aborted_updates = Vec::new(); + + // We skip the first pending update as it is currently being processed. + for result in self.pending_meta.iter(&wtxn)?.skip(1) { + let (key, pending) = result?; + let id = key.get(); + aborted_updates.push((id, pending.abort())); + } + + for (id, aborted) in &aborted_updates { + let key = BEU64::new(*id); + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + } + + wtxn.commit()?; + + Ok(aborted_updates) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::{Duration, Instant}; + + impl HandleUpdate for F + where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { + fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { + self(meta, content) + } + } + + #[test] + fn simple() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let meta = String::from("kiki"); + let update = update_store.register_update(meta, &[]).unwrap(); + thread::sleep(Duration::from_millis(100)); + let meta = update_store.meta(update.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + } + + #[test] + #[ignore] + fn long_running_update() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let before_register = Instant::now(); + + let meta = String::from("kiki"); + let update_kiki = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("coco"); + let update_coco = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("cucu"); + let update_cucu = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + thread::sleep(Duration::from_millis(400 * 3 + 100)); + + let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "coco processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "cucu processed"); + } else { + panic!() + } + } +} diff --git a/src/index_controller/actor_index_controller/uuid_resolver.rs b/src/index_controller/actor_index_controller/uuid_resolver.rs index b75c0402c..3143ae8fc 100644 --- a/src/index_controller/actor_index_controller/uuid_resolver.rs +++ b/src/index_controller/actor_index_controller/uuid_resolver.rs @@ -4,7 +4,7 @@ use uuid::Uuid; use std::collections::HashMap; use std::sync::Arc; use std::collections::hash_map::Entry; -use log::info; +use log::{info, warn}; pub type Result = std::result::Result; @@ -22,8 +22,6 @@ enum UuidResolveMsg { name: String, ret: oneshot::Sender>, }, - Shutdown, - } struct UuidResolverActor { @@ -46,11 +44,13 @@ impl UuidResolverActor { match self.inbox.recv().await { Some(Create { name, ret }) => self.handle_create(name, ret).await, Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, - Some(_) => {} + Some(Resolve { name, ret }) => self.handle_resolve(name, ret).await, // all senders have been dropped, need to quit. None => break, } } + + warn!("exiting uuid resolver loop"); } async fn handle_create(&self, name: String, ret: oneshot::Sender>) { @@ -62,6 +62,11 @@ impl UuidResolverActor { let result = self.store.create_uuid(name, false).await; let _ = ret.send(result); } + + async fn handle_resolve(&self, name: String, ret: oneshot::Sender>>) { + let result = self.store.get_uuid(name).await; + let _ = ret.send(result); + } } #[derive(Clone)] diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index dc6cc3863..cee276ea0 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -13,6 +13,8 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; use tokio::fs::File; +use crate::data::SearchResult; +use crate::data::SearchQuery; pub use updates::{Processed, Processing, Failed}; @@ -135,6 +137,8 @@ pub trait IndexController { primary_key: Option, ) -> anyhow::Result; + async fn search(&self, name: String, query: SearchQuery) -> Result; + /// Clear all documents in the given index. fn clear_documents(&self, index: String) -> anyhow::Result; diff --git a/src/routes/search.rs b/src/routes/search.rs index 7919c5412..1f4218555 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -83,7 +83,7 @@ async fn search_with_url_query( return Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } }; - let search_result = data.search(&path.index_uid, query); + let search_result = data.search(&path.index_uid, query).await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); @@ -101,7 +101,7 @@ async fn search_with_post( path: web::Path, params: web::Json, ) -> Result { - let search_result = data.search(&path.index_uid, params.into_inner()); + let search_result = data.search(&path.index_uid, params.into_inner()).await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); From 1968bfac4d475ad56d02289de08cd5b47ec831be Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Mar 2021 15:48:42 +0100 Subject: [PATCH 100/522] remove legacy tests --- meilisearch-http/_tests/dashboard.rs | 12 - meilisearch-http/_tests/documents_add.rs | 222 -- meilisearch-http/_tests/documents_delete.rs | 67 - meilisearch-http/_tests/documents_get.rs | 23 - meilisearch-http/_tests/dump.rs | 395 ---- meilisearch-http/_tests/errors.rs | 200 -- meilisearch-http/_tests/health.rs | 11 - meilisearch-http/_tests/index.rs | 809 ------- meilisearch-http/_tests/index_update.rs | 200 -- .../_tests/lazy_index_creation.rs | 446 ---- meilisearch-http/_tests/placeholder_search.rs | 629 ------ meilisearch-http/_tests/search.rs | 1879 ----------------- meilisearch-http/_tests/search_settings.rs | 538 ----- meilisearch-http/_tests/settings.rs | 523 ----- .../_tests/settings_ranking_rules.rs | 182 -- .../_tests/settings_stop_words.rs | 61 - meilisearch-http/_tests/url_normalizer.rs | 18 - 17 files changed, 6215 deletions(-) delete mode 100644 meilisearch-http/_tests/dashboard.rs delete mode 100644 meilisearch-http/_tests/documents_add.rs delete mode 100644 meilisearch-http/_tests/documents_delete.rs delete mode 100644 meilisearch-http/_tests/documents_get.rs delete mode 100644 meilisearch-http/_tests/dump.rs delete mode 100644 meilisearch-http/_tests/errors.rs delete mode 100644 meilisearch-http/_tests/health.rs delete mode 100644 meilisearch-http/_tests/index.rs delete mode 100644 meilisearch-http/_tests/index_update.rs delete mode 100644 meilisearch-http/_tests/lazy_index_creation.rs delete mode 100644 meilisearch-http/_tests/placeholder_search.rs delete mode 100644 meilisearch-http/_tests/search.rs delete mode 100644 meilisearch-http/_tests/search_settings.rs delete mode 100644 meilisearch-http/_tests/settings.rs delete mode 100644 meilisearch-http/_tests/settings_ranking_rules.rs delete mode 100644 meilisearch-http/_tests/settings_stop_words.rs delete mode 100644 meilisearch-http/_tests/url_normalizer.rs diff --git a/meilisearch-http/_tests/dashboard.rs b/meilisearch-http/_tests/dashboard.rs deleted file mode 100644 index 2dbaf8f7d..000000000 --- a/meilisearch-http/_tests/dashboard.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn dashboard() { - let mut server = common::Server::with_uid("movies"); - - let (_response, status_code) = server.get_request("/").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("/bulma.min.css").await; - assert_eq!(status_code, 200); -} diff --git a/meilisearch-http/_tests/documents_add.rs b/meilisearch-http/_tests/documents_add.rs deleted file mode 100644 index 382a1ed43..000000000 --- a/meilisearch-http/_tests/documents_add.rs +++ /dev/null @@ -1,222 +0,0 @@ -use serde_json::json; - -mod common; - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/519 -#[actix_rt::test] -async fn check_add_documents_with_primary_key_param() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add documents - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/568 -#[actix_rt::test] -async fn check_add_documents_with_nested_boolean() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a boolean in a nested object - - let body = json!([{ - "id": 12161, - "created_at": "2019-04-10T14:57:57.522Z", - "foo": { - "bar": { - "id": 121, - "crash": false - }, - "id": 45912 - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/571 -#[actix_rt::test] -async fn check_add_documents_with_nested_null() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a null in a nested object - - let body = json!([{ - "id": 0, - "foo": { - "bar": null - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/574 -#[actix_rt::test] -async fn check_add_documents_with_nested_sequence() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a seq in a nested object - - let body = json!([{ - "id": 0, - "foo": { - "bar": [123,456], - "fez": [{ - "id": 255, - "baz": "leesz", - "fuzz": { - "fax": [234] - }, - "sas": [] - }], - "foz": [{ - "id": 255, - "baz": "leesz", - "fuzz": { - "fax": [234] - }, - "sas": [] - }, - { - "id": 256, - "baz": "loss", - "fuzz": { - "fax": [235] - }, - "sas": [321, 321] - }] - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body.clone()).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); - - let url = "/indexes/tasks/search?q=leesz"; - let (response, status_code) = server.get_request(&url).await; - assert_eq!(status_code, 200); - assert_eq!(response["hits"], body); -} - -#[actix_rt::test] -// test sample from #807 -async fn add_document_with_long_field() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let body = json!([{ - "documentId":"de1c2adbb897effdfe0deae32a01035e46f932ce", - "rank":1, - "relurl":"/configuration/app/web.html#locations", - "section":"Web", - "site":"docs", - "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", - "title":"Locations", - "url":"/configuration/app/web.html#locations" - }]); - server.add_or_replace_multiple_documents(body).await; - let (response, _status) = server - .search_post(json!({ "q": "request_buffering" })) - .await; - assert!(!response["hits"].as_array().unwrap().is_empty()); -} - -#[actix_rt::test] -async fn documents_with_same_id_are_overwritten() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test"})).await; - let documents = json!([ - { - "id": 1, - "content": "test1" - }, - { - "id": 1, - "content": "test2" - }, - ]); - server.add_or_replace_multiple_documents(documents).await; - let (response, _status) = server.get_all_documents().await; - assert_eq!(response.as_array().unwrap().len(), 1); - assert_eq!( - response.as_array().unwrap()[0].as_object().unwrap()["content"], - "test2" - ); -} diff --git a/meilisearch-http/_tests/documents_delete.rs b/meilisearch-http/_tests/documents_delete.rs deleted file mode 100644 index 4353a5355..000000000 --- a/meilisearch-http/_tests/documents_delete.rs +++ /dev/null @@ -1,67 +0,0 @@ -mod common; - -use serde_json::json; - -#[actix_rt::test] -async fn delete() { - let mut server = common::Server::test_server().await; - - let (_response, status_code) = server.get_document(50).await; - assert_eq!(status_code, 200); - - server.delete_document(50).await; - - let (_response, status_code) = server.get_document(50).await; - assert_eq!(status_code, 404); -} - -// Resolve the issue https://github.com/meilisearch/MeiliSearch/issues/493 -#[actix_rt::test] -async fn delete_batch() { - let mut server = common::Server::test_server().await; - - let doc_ids = vec!(50, 55, 60); - for doc_id in &doc_ids { - let (_response, status_code) = server.get_document(doc_id).await; - assert_eq!(status_code, 200); - } - - let body = serde_json::json!(&doc_ids); - server.delete_multiple_documents(body).await; - - for doc_id in &doc_ids { - let (_response, status_code) = server.get_document(doc_id).await; - assert_eq!(status_code, 404); - } -} - -#[actix_rt::test] -async fn text_clear_all_placeholder_search() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - - server.create_index(body).await; - let settings = json!({ - "attributesForFaceting": ["genre"], - }); - - server.update_all_settings(settings).await; - - let documents = json!([ - { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance" }, - { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" }, - { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy" }, - { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, - { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, - { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams" } - ]); - - server.add_or_update_multiple_documents(documents).await; - server.clear_all_documents().await; - let (response, _) = server.search_post(json!({ "q": "", "facetsDistribution": ["genre"] })).await; - assert_eq!(response["nbHits"], 0); - let (response, _) = server.search_post(json!({ "q": "" })).await; - assert_eq!(response["nbHits"], 0); -} diff --git a/meilisearch-http/_tests/documents_get.rs b/meilisearch-http/_tests/documents_get.rs deleted file mode 100644 index 35e04f494..000000000 --- a/meilisearch-http/_tests/documents_get.rs +++ /dev/null @@ -1,23 +0,0 @@ -use serde_json::json; -use actix_web::http::StatusCode; - -mod common; - -#[actix_rt::test] -async fn get_documents_from_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.get_all_documents().await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); - assert_eq!(response["errorLink"], "https://docs.meilisearch.com/errors#index_not_found"); -} - -#[actix_rt::test] -async fn get_empty_documents_list() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let (response, status) = server.get_all_documents().await; - assert_eq!(status, StatusCode::OK); - assert!(response.as_array().unwrap().is_empty()); -} diff --git a/meilisearch-http/_tests/dump.rs b/meilisearch-http/_tests/dump.rs deleted file mode 100644 index 701b754aa..000000000 --- a/meilisearch-http/_tests/dump.rs +++ /dev/null @@ -1,395 +0,0 @@ -use assert_json_diff::{assert_json_eq, assert_json_include}; -use meilisearch_http::helpers::compression; -use serde_json::{json, Value}; -use std::fs::File; -use std::path::Path; -use std::thread; -use std::time::Duration; -use tempfile::TempDir; - -#[macro_use] mod common; - -async fn trigger_and_wait_dump(server: &mut common::Server) -> String { - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - for _ in 0..20 as u8 { - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - assert_ne!(value["status"].as_str(), Some("dump_process_failed")); - - if value["status"].as_str() == Some("done") { return dump_uid } - thread::sleep(Duration::from_millis(100)); - } - - unreachable!("dump creation runned out of time") -} - -fn current_db_version() -> (String, String, String) { - let current_version_major = env!("CARGO_PKG_VERSION_MAJOR").to_string(); - let current_version_minor = env!("CARGO_PKG_VERSION_MINOR").to_string(); - let current_version_patch = env!("CARGO_PKG_VERSION_PATCH").to_string(); - - (current_version_major, current_version_minor, current_version_patch) -} - -fn current_dump_version() -> String { - "V1".into() -} - -fn read_all_jsonline(r: R) -> Value { - let deserializer = serde_json::Deserializer::from_reader(r); - let iterator = deserializer.into_iter::(); - - json!(iterator.map(|v| v.unwrap()).collect::>()) -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_should_return_ok() { - let server = common::Server::test_server().await; - - let (_, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_twice_should_return_conflict() { - let server = common::Server::test_server().await; - - let expected = json!({ - "message": "Another dump is already in progress", - "errorCode": "dump_already_in_progress", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" - }); - - let (_, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let (value, status_code) = server.trigger_dump().await; - - - assert_json_eq!(expected, value, ordered: false); - assert_eq!(status_code, 409); -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_concurently_should_return_conflict() { - let server = common::Server::test_server().await; - - let expected = json!({ - "message": "Another dump is already in progress", - "errorCode": "dump_already_in_progress", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" - }); - - let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump()); - - assert_json_eq!(expected, value_2, ordered: false); - assert_eq!(status_code_2, 409); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_early_should_return_in_progress() { - let mut server = common::Server::test_server().await; - - - - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - let expected = json!({ - "uid": dump_uid, - "status": "in_progress" - }); - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_should_return_done() { - let mut server = common::Server::test_server().await; - - - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let expected = json!({ - "uid": dump_uid.clone(), - "status": "done" - }); - - thread::sleep(Duration::from_secs(1)); // wait dump until process end - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_should_return_error_provoking_it() { - let mut server = common::Server::test_server().await; - - - let (value, status_code) = server.trigger_dump().await; - - // removing destination directory provoking `No such file or directory` error - std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap(); - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let expected = json!({ - "uid": dump_uid.clone(), - "status": "failed", - "message": "Dump process failed: compressing dump; No such file or directory (os error 2)", - "errorCode": "dump_process_failed", - "errorType": "internal_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed" - }); - - thread::sleep(Duration::from_secs(1)); // wait dump until process end - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_metadata_should_be_valid() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "uid": "test2", - "primaryKey": "test2_id", - }); - - server.create_index(body).await; - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); - let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); - - // fields are randomly ordered - metadata.get_mut("indexes").unwrap() - .as_array_mut().unwrap() - .sort_by(|a, b| - a.get("uid").unwrap().as_str().cmp(&b.get("uid").unwrap().as_str()) - ); - - let (major, minor, patch) = current_db_version(); - - let expected = json!({ - "indexes": [{ - "uid": "test", - "primaryKey": "id", - }, { - "uid": "test2", - "primaryKey": "test2_id", - } - ], - "dbVersion": format!("{}.{}.{}", major, minor, patch), - "dumpVersion": current_dump_version() - }); - - assert_json_include!(expected: expected, actual: metadata); -} - -#[actix_rt::test] -#[ignore] -async fn dump_gzip_should_have_been_created() { - let mut server = common::Server::test_server().await; - - - let dump_uid = trigger_and_wait_dump(&mut server).await; - let dumps_dir = Path::new(&server.data().dumps_dir); - - let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid)); - assert!(File::open(compressed_path).is_ok()); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_settings_should_be_valid() { - let mut server = common::Server::test_server().await; - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": "email", - "searchableAttributes": [ - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "displayedAttributes": [ - "id", - "isActive", - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "stopWords": [ - "in", - "ad" - ], - "synonyms": { - "wolverine": ["xmen", "logan"], - "logan": ["wolverine", "xmen"] - }, - "attributesForFaceting": [ - "gender", - "color", - "tags" - ] - }); - - server.update_all_settings(expected.clone()).await; - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); - let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); - - assert_json_eq!(expected, settings, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_documents_should_be_valid() { - let mut server = common::Server::test_server().await; - - let dataset = include_bytes!("assets/dumps/v1/test/documents.jsonl"); - let mut slice: &[u8] = dataset; - - let expected: Value = read_all_jsonline(&mut slice); - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); - let documents = read_all_jsonline(file); - - assert_json_eq!(expected, documents, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_updates_should_be_valid() { - let mut server = common::Server::test_server().await; - - let dataset = include_bytes!("assets/dumps/v1/test/updates.jsonl"); - let mut slice: &[u8] = dataset; - - let expected: Value = read_all_jsonline(&mut slice); - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); - let mut updates = read_all_jsonline(file); - - - // hotfix until #943 is fixed (https://github.com/meilisearch/MeiliSearch/issues/943) - updates.as_array_mut().unwrap() - .get_mut(0).unwrap() - .get_mut("type").unwrap() - .get_mut("settings").unwrap() - .get_mut("displayed_attributes").unwrap() - .get_mut("Update").unwrap() - .as_array_mut().unwrap().sort_by(|a, b| a.as_str().cmp(&b.as_str())); - - eprintln!("{}\n", updates.to_string()); - eprintln!("{}", expected.to_string()); - assert_json_include!(expected: expected, actual: updates); -} - -#[actix_rt::test] -#[ignore] -async fn get_unexisting_dump_status_should_return_not_found() { - let mut server = common::Server::test_server().await; - - let (_, status_code) = server.get_dump_status("4242").await; - - assert_eq!(status_code, 404); -} diff --git a/meilisearch-http/_tests/errors.rs b/meilisearch-http/_tests/errors.rs deleted file mode 100644 index e11483356..000000000 --- a/meilisearch-http/_tests/errors.rs +++ /dev/null @@ -1,200 +0,0 @@ -mod common; - -use std::thread; -use std::time::Duration; - -use actix_http::http::StatusCode; -use serde_json::{json, Map, Value}; - -macro_rules! assert_error { - ($code:literal, $type:literal, $status:path, $req:expr) => { - let (response, status_code) = $req; - assert_eq!(status_code, $status); - assert_eq!(response["errorCode"].as_str().unwrap(), $code); - assert_eq!(response["errorType"].as_str().unwrap(), $type); - }; -} - -macro_rules! assert_error_async { - ($code:literal, $type:literal, $server:expr, $req:expr) => { - let (response, _) = $req; - let update_id = response["updateId"].as_u64().unwrap(); - for _ in 1..10 { - let (response, status_code) = $server.get_update_status(update_id).await; - assert_eq!(status_code, StatusCode::OK); - if response["status"] == "processed" || response["status"] == "failed" { - println!("response: {}", response); - assert_eq!(response["status"], "failed"); - assert_eq!(response["errorCode"], $code); - assert_eq!(response["errorType"], $type); - return - } - thread::sleep(Duration::from_secs(1)); - } - }; -} - -#[actix_rt::test] -async fn index_already_exists_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test" - }); - let (response, status_code) = server.create_index(body.clone()).await; - println!("{}", response); - assert_eq!(status_code, StatusCode::CREATED); - - let (response, status_code) = server.create_index(body.clone()).await; - println!("{}", response); - - assert_error!( - "index_already_exists", - "invalid_request_error", - StatusCode::BAD_REQUEST, - (response, status_code)); -} - -#[actix_rt::test] -async fn index_not_found_error() { - let mut server = common::Server::with_uid("test"); - assert_error!( - "index_not_found", - "invalid_request_error", - StatusCode::NOT_FOUND, - server.get_index().await); -} - -#[actix_rt::test] -async fn primary_key_already_present_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "test" - }); - server.create_index(body.clone()).await; - let body = json!({ - "primaryKey": "t" - }); - assert_error!( - "primary_key_already_present", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.update_index(body).await); -} - -#[actix_rt::test] -async fn max_field_limit_exceeded_error() { - let mut server = common::Server::test_server().await; - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - let mut doc = Map::with_capacity(70_000); - doc.insert("id".into(), Value::String("foo".into())); - for i in 0..69_999 { - doc.insert(format!("field{}", i), Value::String("foo".into())); - } - let docs = json!([doc]); - assert_error_async!( - "max_fields_limit_exceeded", - "invalid_request_error", - server, - server.add_or_replace_multiple_documents_sync(docs).await); -} - -#[actix_rt::test] -async fn missing_document_id() { - let mut server = common::Server::test_server().await; - let body = json!({ - "uid": "test", - "primaryKey": "test" - }); - server.create_index(body).await; - let docs = json!([ - { - "foo": "bar", - } - ]); - assert_error_async!( - "missing_document_id", - "invalid_request_error", - server, - server.add_or_replace_multiple_documents_sync(docs).await); -} - -#[actix_rt::test] -async fn facet_error() { - let mut server = common::Server::test_server().await; - let search = json!({ - "q": "foo", - "facetFilters": ["test:hello"] - }); - assert_error!( - "invalid_facet", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(search).await); -} - -#[actix_rt::test] -async fn filters_error() { - let mut server = common::Server::test_server().await; - let search = json!({ - "q": "foo", - "filters": "fo:12" - }); - assert_error!( - "invalid_filter", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(search).await); -} - -#[actix_rt::test] -async fn bad_request_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "foo": "bar", - }); - assert_error!( - "bad_request", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(body).await); -} - -#[actix_rt::test] -async fn document_not_found_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({"uid": "test"})).await; - assert_error!( - "document_not_found", - "invalid_request_error", - StatusCode::NOT_FOUND, - server.get_document(100).await); -} - -#[actix_rt::test] -async fn payload_too_large_error() { - let mut server = common::Server::with_uid("test"); - let bigvec = vec![0u64; 10_000_000]; // 80mb - assert_error!( - "payload_too_large", - "invalid_request_error", - StatusCode::PAYLOAD_TOO_LARGE, - server.create_index(json!(bigvec)).await); -} - -#[actix_rt::test] -async fn missing_primary_key_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({"uid": "test"})).await; - let document = json!([{ - "content": "test" - }]); - assert_error!( - "missing_primary_key", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.add_or_replace_multiple_documents_sync(document).await); -} diff --git a/meilisearch-http/_tests/health.rs b/meilisearch-http/_tests/health.rs deleted file mode 100644 index f72127431..000000000 --- a/meilisearch-http/_tests/health.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn test_healthyness() { - let mut server = common::Server::with_uid("movies"); - - // Check that the server is healthy - - let (_response, status_code) = server.get_health().await; - assert_eq!(status_code, 204); -} diff --git a/meilisearch-http/_tests/index.rs b/meilisearch-http/_tests/index.rs deleted file mode 100644 index 271507e03..000000000 --- a/meilisearch-http/_tests/index.rs +++ /dev/null @@ -1,809 +0,0 @@ -use actix_web::http::StatusCode; -use assert_json_diff::assert_json_eq; -use serde_json::{json, Value}; - -mod common; - -#[actix_rt::test] -async fn create_index_with_name() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn create_index_with_uid() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body.clone()).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid, "movies"); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 1.5 verify that error is thrown when trying to create the same index - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - assert_eq!( - response["errorCode"].as_str().unwrap(), - "index_already_exists" - ); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn create_index_with_name_and_uid() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "Films", - "uid": "fr_movies", - }); - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "Films"); - assert_eq!(r1_uid, "fr_movies"); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn rename_index() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 6); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Update an index name - - let body = json!({ - "name": "TV Shows", - }); - - let (res2_value, status_code) = server.update_index(body).await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_object().unwrap().len(), 5); - let r2_name = res2_value["name"].as_str().unwrap(); - let r2_uid = res2_value["uid"].as_str().unwrap(); - let r2_created_at = res2_value["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, "TV Shows"); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at, r1_created_at); - assert!(r2_updated_at.len() > 1); - - // 3 - Check the list of indexes - - let (res3_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res3_value.as_array().unwrap().len(), 1); - assert_eq!(res3_value[0].as_object().unwrap().len(), 5); - let r3_name = res3_value[0]["name"].as_str().unwrap(); - let r3_uid = res3_value[0]["uid"].as_str().unwrap(); - let r3_created_at = res3_value[0]["createdAt"].as_str().unwrap(); - let r3_updated_at = res3_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r3_name, r2_name); - assert_eq!(r3_uid.len(), r1_uid.len()); - assert_eq!(r3_created_at.len(), r1_created_at.len()); - assert_eq!(r3_updated_at.len(), r2_updated_at.len()); -} - -#[actix_rt::test] -async fn delete_index_and_recreate_it() { - let mut server = common::Server::with_uid("movies"); - - // 0 - delete unexisting index is error - - let (response, status_code) = server.delete_request("/indexes/test").await; - assert_eq!(status_code, 404); - assert_eq!(&response["errorCode"], "index_not_found"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 6); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); - - // 3- Delete an index - - let (_res2_value, status_code) = server.delete_index().await; - - assert_eq!(status_code, 204); - - // 4 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 0); - - // 5 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 6 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn check_multiples_indexes() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_0_name = res2_value[0]["name"].as_str().unwrap(); - let r2_0_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_0_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_0_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_0_name, r1_name); - assert_eq!(r2_0_uid.len(), r1_uid.len()); - assert_eq!(r2_0_created_at.len(), r1_created_at.len()); - assert_eq!(r2_0_updated_at.len(), r1_updated_at.len()); - - // 3 - Create a new index - - let body = json!({ - "name": "films", - }); - - let (res3_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res3_value.as_object().unwrap().len(), 5); - let r3_name = res3_value["name"].as_str().unwrap(); - let r3_uid = res3_value["uid"].as_str().unwrap(); - let r3_created_at = res3_value["createdAt"].as_str().unwrap(); - let r3_updated_at = res3_value["updatedAt"].as_str().unwrap(); - assert_eq!(r3_name, "films"); - assert_eq!(r3_uid.len(), 8); - assert!(r3_created_at.len() > 1); - assert!(r3_updated_at.len() > 1); - - // 4 - Check the list of indexes - - let (res4_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res4_value.as_array().unwrap().len(), 2); - assert_eq!(res4_value[0].as_object().unwrap().len(), 5); - let r4_0_name = res4_value[0]["name"].as_str().unwrap(); - let r4_0_uid = res4_value[0]["uid"].as_str().unwrap(); - let r4_0_created_at = res4_value[0]["createdAt"].as_str().unwrap(); - let r4_0_updated_at = res4_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(res4_value[1].as_object().unwrap().len(), 5); - let r4_1_name = res4_value[1]["name"].as_str().unwrap(); - let r4_1_uid = res4_value[1]["uid"].as_str().unwrap(); - let r4_1_created_at = res4_value[1]["createdAt"].as_str().unwrap(); - let r4_1_updated_at = res4_value[1]["updatedAt"].as_str().unwrap(); - if r4_0_name == r1_name { - assert_eq!(r4_0_name, r1_name); - assert_eq!(r4_0_uid.len(), r1_uid.len()); - assert_eq!(r4_0_created_at.len(), r1_created_at.len()); - assert_eq!(r4_0_updated_at.len(), r1_updated_at.len()); - } else { - assert_eq!(r4_0_name, r3_name); - assert_eq!(r4_0_uid.len(), r3_uid.len()); - assert_eq!(r4_0_created_at.len(), r3_created_at.len()); - assert_eq!(r4_0_updated_at.len(), r3_updated_at.len()); - } - if r4_1_name == r1_name { - assert_eq!(r4_1_name, r1_name); - assert_eq!(r4_1_uid.len(), r1_uid.len()); - assert_eq!(r4_1_created_at.len(), r1_created_at.len()); - assert_eq!(r4_1_updated_at.len(), r1_updated_at.len()); - } else { - assert_eq!(r4_1_name, r3_name); - assert_eq!(r4_1_uid.len(), r3_uid.len()); - assert_eq!(r4_1_created_at.len(), r3_created_at.len()); - assert_eq!(r4_1_updated_at.len(), r3_updated_at.len()); - } -} - -#[actix_rt::test] -async fn create_index_failed() { - let mut server = common::Server::with_uid("movies"); - - // 2 - Push index creation with empty json body - - let body = json!({}); - - let (res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = res_value["message"].as_str().unwrap(); - assert_eq!(res_value.as_object().unwrap().len(), 4); - assert_eq!(message, "Index creation must have an uid"); - - // 3 - Create a index with extra data - - let body = json!({ - "name": "movies", - "active": true - }); - - let (_res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - - // 3 - Create a index with wrong data type - - let body = json!({ - "name": "movies", - "uid": 0 - }); - - let (_res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); -} - -// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/492 -#[actix_rt::test] -async fn create_index_with_primary_key_and_index() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index - - let body = json!({ - "uid": "movies", - "primaryKey": "id", - }); - - let (_response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - - // 2 - Add content - - let body = json!([{ - "id": 123, - "text": "The mask" - }]); - - server.add_or_replace_multiple_documents(body.clone()).await; - - // 3 - Retreive document - - let (response, _status_code) = server.get_document(123).await; - - let expect = json!({ - "id": 123, - "text": "The mask" - }); - - assert_json_eq!(response, expect, ordered: false); -} - -// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/497 -// Test when the given index uid is not valid -// Should have a 400 status code -// Should have the right error message -#[actix_rt::test] -async fn create_index_with_invalid_uid() { - let mut server = common::Server::with_uid(""); - - // 1 - Create the index with invalid uid - - let body = json!({ - "uid": "the movies" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 2 - Create the index with invalid uid - - let body = json!({ - "uid": "%$#" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 3 - Create the index with invalid uid - - let body = json!({ - "uid": "the~movies" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 4 - Create the index with invalid uid - - let body = json!({ - "uid": "🎉" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); -} - -// Test that it's possible to add primary_key if it's not already set on index creation -#[actix_rt::test] -async fn create_index_and_add_indentifier_after() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Update the index and add an primary_key. - - let body = json!({ - "primaryKey": "id", - }); - - let (response, status_code) = server.update_index(body).await; - assert_eq!(status_code, 200); - eprintln!("response: {:#?}", response); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); - - // 3 - Get index to verify if the primary_key is good - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test that it's impossible to change the primary_key -#[actix_rt::test] -async fn create_index_and_update_indentifier_after() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - "primaryKey": "id", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); - - // 2 - Update the index and add an primary_key. - - let body = json!({ - "primaryKey": "skuid", - }); - - let (_response, status_code) = server.update_index(body).await; - assert_eq!(status_code, 400); - - // 3 - Get index to verify if the primary_key still the first one - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test that schema inference work well -#[actix_rt::test] -async fn create_index_without_primary_key_and_add_document() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document - - let body = json!([{ - "id": 123, - "title": "I'm a legend", - }]); - - server.add_or_update_multiple_documents(body).await; - - // 3 - Get index to verify if the primary_key is good - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test search with no primary_key -#[actix_rt::test] -async fn create_index_without_primary_key_and_search() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Search - - let query = "q=captain&limit=3"; - - let (response, status_code) = server.search_get(&query).await; - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 0); -} - -// Test the error message when we push an document update and impossibility to find primary key -// Test issue https://github.com/meilisearch/MeiliSearch/issues/517 -#[actix_rt::test] -async fn check_add_documents_without_primary_key() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2- Add document - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let (response, status_code) = server.add_or_replace_multiple_documents_sync(body).await; - - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(response["errorCode"], "missing_primary_key"); - assert_eq!(status_code, 400); -} - -#[actix_rt::test] -async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { - let mut server = common::Server::with_uid("movies"); - - let body = json!({ - "uid": "movies", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let dataset = include_bytes!("./assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - // 2. Index the documents from movies.json, present inside of assets directory - server.add_or_replace_multiple_documents(body).await; - - // 3. Fetch the status of the indexing done above. - let (response, status_code) = server.get_all_updates_status().await; - - // 4. Verify the fetch is successful and indexing status is 'processed' - assert_eq!(status_code, 200); - assert_eq!(response[0]["status"], "processed"); -} - -#[actix_rt::test] -async fn get_empty_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server.list_indexes().await; - assert!(response.as_array().unwrap().is_empty()); -} - -#[actix_rt::test] -async fn create_and_list_multiple_indices() { - let mut server = common::Server::with_uid("test"); - for i in 0..10 { - server - .create_index(json!({ "uid": format!("test{}", i) })) - .await; - } - let (response, _status) = server.list_indexes().await; - assert_eq!(response.as_array().unwrap().len(), 10); -} - -#[actix_rt::test] -async fn get_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.get_index().await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn create_index_twice_is_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let (response, status) = server.create_index(json!({ "uid": "test" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "index_already_exists"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn badly_formatted_index_name_is_error() { - let mut server = common::Server::with_uid("$__test"); - let (response, status) = server.create_index(json!({ "uid": "$__test" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "invalid_index_uid"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn correct_response_no_primary_key_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server.create_index(json!({ "uid": "test" })).await; - assert_eq!(response["primaryKey"], Value::Null); -} - -#[actix_rt::test] -async fn correct_response_with_primary_key_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server - .create_index(json!({ "uid": "test", "primaryKey": "test" })) - .await; - assert_eq!(response["primaryKey"], "test"); -} - -#[actix_rt::test] -async fn udpate_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.update_index(json!({ "primaryKey": "foobar" })).await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn update_existing_primary_key_is_error() { - let mut server = common::Server::with_uid("test"); - server - .create_index(json!({ "uid": "test", "primaryKey": "key" })) - .await; - let (response, status) = server.update_index(json!({ "primaryKey": "test2" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "primary_key_already_present"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn test_facets_distribution_attribute() { - let mut server = common::Server::test_server().await; - - let (response, _status_code) = server.get_index_stats().await; - - let expected = json!({ - "isIndexing": false, - "numberOfDocuments":77, - "fieldsDistribution":{ - "age":77, - "gender":77, - "phone":77, - "name":77, - "registered":77, - "latitude":77, - "email":77, - "tags":77, - "longitude":77, - "color":77, - "address":77, - "balance":77, - "about":77, - "picture":77, - }, - }); - - assert_json_eq!(expected, response, ordered: true); -} diff --git a/meilisearch-http/_tests/index_update.rs b/meilisearch-http/_tests/index_update.rs deleted file mode 100644 index df4639252..000000000 --- a/meilisearch-http/_tests/index_update.rs +++ /dev/null @@ -1,200 +0,0 @@ -use serde_json::json; -use serde_json::Value; -use assert_json_diff::assert_json_include; - -mod common; - -#[actix_rt::test] -async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let dataset = include_bytes!("assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - // 2. Index the documents from movies.json, present inside of assets directory - server.add_or_replace_multiple_documents(body).await; - - // 3. Fetch the status of the indexing done above. - let (response, status_code) = server.get_all_updates_status().await; - - // 4. Verify the fetch is successful and indexing status is 'processed' - assert_eq!(status_code, 200); - assert_eq!(response[0]["status"], "processed"); -} - -#[actix_rt::test] -async fn return_error_when_get_update_status_of_unexisting_index() { - let mut server = common::Server::with_uid("test"); - - // 1. Fetch the status of unexisting index. - let (_, status_code) = server.get_all_updates_status().await; - - // 2. Verify the fetch returned 404 - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn return_empty_when_get_update_status_of_empty_index() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2. Fetch the status of empty index. - let (response, status_code) = server.get_all_updates_status().await; - - // 3. Verify the fetch is successful, and no document are returned - assert_eq!(status_code, 200); - assert_eq!(response, json!([])); -} - -#[actix_rt::test] -async fn return_update_status_of_pushed_documents() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - - let bodies = vec![ - json!([{ - "title": "Test", - "comment": "comment test" - }]), - json!([{ - "title": "Test1", - "comment": "comment test1" - }]), - json!([{ - "title": "Test2", - "comment": "comment test2" - }]), - ]; - - let mut update_ids = Vec::new(); - - let url = "/indexes/test/documents?primaryKey=title"; - for body in bodies { - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - update_ids.push(update_id); - } - - // 2. Fetch the status of index. - let (response, status_code) = server.get_all_updates_status().await; - - // 3. Verify the fetch is successful, and updates are returned - - let expected = json!([{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[0] - },{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[1] - },{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[2] - },]); - - assert_eq!(status_code, 200); - assert_json_include!(actual: json!(response), expected: expected); -} - -#[actix_rt::test] -async fn return_error_if_index_does_not_exist() { - let mut server = common::Server::with_uid("test"); - - let (response, status_code) = server.get_update_status(42).await; - - assert_eq!(status_code, 404); - assert_eq!(response["errorCode"], "index_not_found"); -} - -#[actix_rt::test] -async fn return_error_if_update_does_not_exist() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let (response, status_code) = server.get_update_status(42).await; - - assert_eq!(status_code, 404); - assert_eq!(response["errorCode"], "not_found"); -} - -#[actix_rt::test] -async fn should_return_existing_update() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/test/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - - let update_id = response["updateId"].as_u64().unwrap(); - - let expected = json!({ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_id - }); - - let (response, status_code) = server.get_update_status(update_id).await; - - assert_eq!(status_code, 200); - assert_json_include!(actual: json!(response), expected: expected); -} diff --git a/meilisearch-http/_tests/lazy_index_creation.rs b/meilisearch-http/_tests/lazy_index_creation.rs deleted file mode 100644 index 6730db82e..000000000 --- a/meilisearch-http/_tests/lazy_index_creation.rs +++ /dev/null @@ -1,446 +0,0 @@ -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Add documents - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents_and_discover_pk() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Add documents - - let body = json!([{ - "id": 1, - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents_with_wrong_name() { - let server = common::Server::with_uid("wrong&name"); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/wrong&name/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_index_uid"); -} - -#[actix_rt::test] -async fn create_index_lazy_add_documents_failed() { - let mut server = common::Server::with_uid("wrong&name"); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/wrong&name/documents"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_index_uid"); - - let (_, status_code) = server.get_index().await; - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_settings() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_settings_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "other", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "anotherSettings": ["name"], - }); - - let (_, status_code) = server.update_all_settings_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_ranking_rules() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_ranking_rules_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": 123, - }); - - let (_, status_code) = server.update_ranking_rules_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_distinct_attribute() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!("type"); - - server.update_distinct_attribute(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_distinct_attribute_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (resp, status_code) = server.update_distinct_attribute_sync(body.clone()).await; - eprintln!("resp: {:?}", resp); - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (resp, status_code) = server.get_all_settings().await; - eprintln!("resp: {:?}", resp); - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_searchable_attributes() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_searchable_attributes(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_searchable_attributes_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_searchable_attributes_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_displayed_attributes() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_displayed_attributes(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_displayed_attributes_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_displayed_attributes_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_attributes_for_faceting() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_attributes_for_faceting(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_attributes_for_faceting_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server - .update_attributes_for_faceting_sync(body.clone()) - .await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_synonyms() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "road": ["street", "avenue"], - "street": ["avenue"], - }); - - server.update_synonyms(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_synonyms_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_synonyms_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_stop_words() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["le", "la", "les"]); - - server.update_stop_words(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_stop_words_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_stop_words_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} diff --git a/meilisearch-http/_tests/placeholder_search.rs b/meilisearch-http/_tests/placeholder_search.rs deleted file mode 100644 index 048ab7f8b..000000000 --- a/meilisearch-http/_tests/placeholder_search.rs +++ /dev/null @@ -1,629 +0,0 @@ -use std::convert::Into; - -use serde_json::json; -use serde_json::Value; -use std::cell::RefCell; -use std::sync::Mutex; - -#[macro_use] -mod common; - -#[actix_rt::test] -async fn placeholder_search_with_limit() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "limit": 3 - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 3); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_offset() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 6, - }); - - // hack to take a value out of macro (must implement UnwindSafe) - let expected = Mutex::new(RefCell::new(Vec::new())); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - // take results at offset 3 as reference - let lock = expected.lock().unwrap(); - lock.replace(response["hits"].as_array().unwrap()[3..6].to_vec()); - }); - let expected = expected.into_inner().unwrap().into_inner(); - - let query = json!({ - "limit": 3, - "offset": 3, - }); - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let response = response["hits"].as_array().unwrap(); - assert_eq!(&expected, response); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_attribute_to_highlight_wildcard() { - // there should be no highlight in placeholder search - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 1, - "attributesToHighlight": ["*"] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let result = response["hits"].as_array().unwrap()[0].as_object().unwrap(); - for value in result.values() { - assert!(value.to_string().find("").is_none()); - } - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_matches() { - // matches is always empty - let mut server = common::Server::test_server().await; - - let query = json!({ - "matches": true - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let result = response["hits"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_object().unwrap()["_matchesInfo"].clone()) - .all(|m| m.as_object().unwrap().is_empty()); - assert!(result); - }); -} - -#[actix_rt::test] -async fn placeholder_search_witch_crop() { - // placeholder search crop always crop from beggining - let mut server = common::Server::test_server().await; - - let query = json!({ - "attributesToCrop": ["about"], - "cropLength": 20 - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - - let hits = response["hits"].as_array().unwrap(); - - for hit in hits { - let hit = hit.as_object().unwrap(); - let formatted = hit["_formatted"].as_object().unwrap(); - - let about = hit["about"].as_str().unwrap(); - let about_formatted = formatted["about"].as_str().unwrap(); - // the formatted about length should be about 20 characters long - assert!(about_formatted.len() < 20 + 10); - // the formatted part should be located at the beginning of the original one - assert_eq!(about.find(&about_formatted).unwrap(), 0); - } - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_attributes_to_retrieve() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 1, - "attributesToRetrieve": ["gender", "about"], - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hit = response["hits"].as_array().unwrap()[0].as_object().unwrap(); - assert_eq!(hit.values().count(), 2); - let _ = hit["gender"]; - let _ = hit["about"]; - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_filter() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "filters": "color='green'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green")); - }); - - let query = json!({ - "filters": "tags=bug" - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let value = Value::String(String::from("bug")); - assert!(hits - .iter() - .all(|v| v["tags"].as_array().unwrap().contains(&value))); - }); - - let query = json!({ - "filters": "color='green' AND (tags='bug' OR tags='wontfix')" - }); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let bug = Value::String(String::from("bug")); - let wontfix = Value::String(String::from("wontfix")); - assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green" - && v["tags"].as_array().unwrap().contains(&bug) - || v["tags"].as_array().unwrap().contains(&wontfix))); - }); -} - -#[actix_rt::test] -async fn placeholder_test_faceted_search_valid() { - let mut server = common::Server::test_server().await; - - // simple tests on attributes with string value - let body = json!({ - "attributesForFaceting": ["color"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "facetFilters": ["color:green"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "Green")); - }); - - let query = json!({ - "facetFilters": [["color:blue"]] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - let query = json!({ - "facetFilters": ["color:Blue"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - // test on arrays: ["tags:bug"] - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "facetFilters": ["tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())))); - }); - - // test and: ["color:blue", "tags:bug"] - let query = json!({ - "facetFilters": ["color:blue", "tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue" - && value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())))); - }); - - // test or: [["color:blue", "color:green"]] - let query = json!({ - "facetFilters": [["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue" - || value.get("color").unwrap() == "Green")); - }); - // test and-or: ["tags:bug", ["color:blue", "color:green"]] - let query = json!({ - "facetFilters": ["tags:bug", ["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())) - && (value.get("color").unwrap() == "blue" - || value.get("color").unwrap() == "Green"))); - }); -} - -#[actix_rt::test] -async fn placeholder_test_faceted_search_invalid() { - let mut server = common::Server::test_server().await; - - //no faceted attributes set - let query = json!({ - "facetFilters": ["color:blue"] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // empty arrays are error - // [] - let query = json!({ - "facetFilters": [] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // [[]] - let query = json!({ - "facetFilters": [[]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // ["color:green", []] - let query = json!({ - "facetFilters": ["color:green", []] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - - // too much depth - // [[[]]] - let query = json!({ - "facetFilters": [[[]]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // [["color:green", ["color:blue"]]] - let query = json!({ - "facetFilters": [["color:green", ["color:blue"]]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // "color:green" - let query = json!({ - "facetFilters": "color:green" - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); -} - -#[actix_rt::test] -async fn placeholder_test_facet_count() { - let mut server = common::Server::test_server().await; - - // test without facet distribution - let query = json!({}); - test_post_get_search!(server, query, |response, _status_code| { - assert!(response.get("exhaustiveFacetsCount").is_none()); - assert!(response.get("facetsDistribution").is_none()); - }); - - // test no facets set, search on color - let query = json!({ - "facetsDistribution": ["color"] - }); - test_post_get_search!(server, query.clone(), |_response, status_code| { - assert_eq!(status_code, 400); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // same as before, but now facets are set: - test_post_get_search!(server, query, |response, _status_code| { - println!("{}", response); - assert!(response.get("exhaustiveFacetsCount").is_some()); - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 1 - ); - }); - // searching on color and tags - let query = json!({ - "facetsDistribution": ["color", "tags"] - }); - test_post_get_search!(server, query, |response, _status_code| { - let facets = response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap(); - assert_eq!(facets.values().count(), 2); - assert_ne!( - !facets - .get("color") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - assert_ne!( - !facets - .get("tags") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - }); - // wildcard - let query = json!({ - "facetsDistribution": ["*"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 2 - ); - }); - // wildcard with other attributes: - let query = json!({ - "facetsDistribution": ["color", "*"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 2 - ); - }); - - // empty facet list - let query = json!({ - "facetsDistribution": [] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - }); - - // attr not set as facet passed: - let query = json!({ - "facetsDistribution": ["gender"] - }); - test_post_get_search!(server, query, |_response, status_code| { - assert_eq!(status_code, 400); - }); -} - -#[actix_rt::test] -#[should_panic] -async fn placeholder_test_bad_facet_distribution() { - let mut server = common::Server::test_server().await; - // string instead of array: - let query = json!({ - "facetsDistribution": "color" - }); - test_post_get_search!(server, query, |_response, _status_code| {}); - - // invalid value in array: - let query = json!({ - "facetsDistribution": ["color", true] - }); - test_post_get_search!(server, query, |_response, _status_code| {}); -} - -#[actix_rt::test] -async fn placeholder_test_sort() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "rankingRules": ["asc(age)"], - "attributesForFaceting": ["color"] - }); - server.update_all_settings(body).await; - let query = json!({}); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - hits.iter() - .map(|v| v["age"].as_u64().unwrap()) - .fold(0, |prev, cur| { - assert!(cur >= prev); - cur - }); - }); - - let query = json!({ - "facetFilters": ["color:green"] - }); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - hits.iter() - .map(|v| v["age"].as_u64().unwrap()) - .fold(0, |prev, cur| { - assert!(cur >= prev); - cur - }); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_empty_query() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "", - "limit": 3 - }); - - test_post_get_search!(server, query, |response, status_code| { - eprintln!("{}", response); - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 3); - }); -} - -#[actix_rt::test] -async fn test_filter_nb_hits_search_placeholder() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let documents = json!([ - { - "id": 1, - "content": "a", - "color": "green", - "size": 1, - }, - { - "id": 2, - "content": "a", - "color": "green", - "size": 2, - }, - { - "id": 3, - "content": "a", - "color": "blue", - "size": 3, - }, - ]); - - server.add_or_update_multiple_documents(documents).await; - let (response, _) = server.search_post(json!({})).await; - assert_eq!(response["nbHits"], 3); - - server.update_distinct_attribute(json!("color")).await; - - let (response, _) = server.search_post(json!({})).await; - assert_eq!(response["nbHits"], 2); - - let (response, _) = server.search_post(json!({"filters": "size < 3"})).await; - println!("result: {}", response); - assert_eq!(response["nbHits"], 1); -} diff --git a/meilisearch-http/_tests/search.rs b/meilisearch-http/_tests/search.rs deleted file mode 100644 index 267c98265..000000000 --- a/meilisearch-http/_tests/search.rs +++ /dev/null @@ -1,1879 +0,0 @@ -use std::convert::Into; - -use assert_json_diff::assert_json_eq; -use serde_json::json; -use serde_json::Value; - -#[macro_use] mod common; - -#[actix_rt::test] -async fn search() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "exercitation" - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let hits: Vec = hits.iter().cloned().take(3).collect(); - assert_json_eq!(expected.clone(), serde_json::to_value(hits).unwrap(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_no_params() { - let mut server = common::Server::test_server().await; - - let query = json! ({}); - - // an empty search should return the 20 first indexed document - let dataset: Vec = serde_json::from_slice(include_bytes!("assets/test_set.json")).unwrap(); - let expected: Vec = dataset.into_iter().take(20).collect(); - let expected: Value = serde_json::to_value(expected).unwrap(); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_in_unexisting_index() { - let mut server = common::Server::with_uid("test"); - - let query = json! ({ - "q": "exercitation" - }); - - let expected = json! ({ - "message": "Index test not found", - "errorCode": "index_not_found", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#index_not_found" - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(404, status_code); - assert_json_eq!(expected.clone(), response.clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_unexpected_params() { - - let query = json! ({"lol": "unexpected"}); - - let expected = "unknown field `lol`, expected one of `q`, `offset`, `limit`, `attributesToRetrieve`, `attributesToCrop`, `cropLength`, `attributesToHighlight`, `filters`, `matches`, `facetFilters`, `facetsDistribution` at line 1 column 6"; - - let post_query = serde_json::from_str::(&query.to_string()); - assert!(post_query.is_err()); - assert_eq!(expected, post_query.err().unwrap().to_string()); - - let get_query: Result = serde_json::from_str(&query.to_string()); - assert!(get_query.is_err()); - assert_eq!(expected, get_query.err().unwrap().to_string()); -} - -#[actix_rt::test] -async fn search_with_limit() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "exercitation", - "limit": 3 - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_offset() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 3, - "offset": 1 - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_wildcard() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["*"] - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_1() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["name"] - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_matches() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "matches": true - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_matchesInfo": { - "name": [ - { - "start": 0, - "length": 6 - } - ], - "email": [ - { - "start": 0, - "length": 6 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_crop() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToCrop": ["about"], - "cropLength": 20 - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_retrieve() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","color","gender"], - }); - - let expected = json!([ - { - "name": "Cherry Orr", - "age": 27, - "color": "Green", - "gender": "female" - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_retrieve_wildcard() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["*"], - }); - - let expected = json!([ - { - "id": 1, - "isActive": true, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ] - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_filter() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "gender='male'" - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - }, - { - "id": 66, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - } - ]); - - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "name='Lucas Hess'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 2, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ], - "isActive": true - }, - { - "id": 75, - "balance": "$1,913.42", - "picture": "http://placehold.it/32x32", - "age": 24, - "color": "Green", - "name": "Emma Jacobs", - "gender": "female", - "email": "emmajacobs@chorizon.com", - "phone": "+1 (899) 554-3847", - "address": "173 Tapscott Street, Esmont, Maine, 7450", - "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", - "registered": "2019-03-29T06:24:13 -01:00", - "latitude": -35.53722, - "longitude": 155.703874, - "tags": [], - "isActive": false - } - ]); - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "gender='female' AND (name='Patricia Goff' OR name='Emma Jacobs')" - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 30, - "balance": "$2,021.11", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Stacy Espinoza", - "gender": "female", - "email": "stacyespinoza@chorizon.com", - "phone": "+1 (999) 487-3253", - "address": "931 Alabama Avenue, Bangor, Alaska, 8215", - "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", - "registered": "2014-07-16T06:15:53 -02:00", - "latitude": 41.560197, - "longitude": 177.697, - "tags": [ - "new issue", - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 31, - "balance": "$3,609.82", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Vilma Garza", - "gender": "female", - "email": "vilmagarza@chorizon.com", - "phone": "+1 (944) 585-2021", - "address": "565 Tech Place, Sedley, Puerto Rico, 858", - "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", - "registered": "2017-06-30T07:43:52 -02:00", - "latitude": -12.574889, - "longitude": -54.771186, - "tags": [ - "new issue", - "wontfix", - "wontfix" - ], - "isActive": false - }, - { - "id": 2, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "limit": 3, - "filters": "gender='female' AND (name='Patricia Goff' OR age > 30)" - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - }, - { - "id": 66, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "limit": 3, - "filters": "NOT gender = 'female' AND age > 30" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 11, - "balance": "$1,351.43", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Evans Wagner", - "gender": "male", - "email": "evanswagner@chorizon.com", - "phone": "+1 (889) 496-2332", - "address": "118 Monaco Place, Lutsen, Delaware, 6209", - "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", - "registered": "2016-10-27T01:26:31 -02:00", - "latitude": -77.673222, - "longitude": -142.657214, - "tags": [ - "good first issue", - "good first issue" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "filters": "NOT gender = 'female' AND name='Evans Wagner'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_highlight_and_matches() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["name","email"], - "matches": true, - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - "_matchesInfo": { - "email": [ - { - "start": 0, - "length": 6 - } - ], - "name": [ - { - "start": 0, - "length": 6 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_highlight_and_matches_and_crop() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exerciatation", - "limit": 1, - "attributesToCrop": ["about"], - "cropLength": 20, - "attributesToHighlight": ["about"], - "matches": true, - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - "_matchesInfo": { - "about": [ - { - "start": 0, - "length": 12 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","gender","email"], - "attributesToHighlight": ["name"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry Orr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_2() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToRetrieve": ["name","age","gender"], - "attributesToCrop": ["about"], - "cropLength": 20, - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "_formatted": { - "about": "Exercitation officia" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_3() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToRetrieve": ["name","age","gender"], - "attributesToCrop": ["about:20"], - }); - - let expected = json!( [ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "_formatted": { - "about": "Exercitation officia" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_4() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["name:0","email:6"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry", - "email": "cherryorr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_5() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["*","email:6"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry Orr", - "email": "cherryorr", - "age": 27, - "gender": "female" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_6() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["*","email:10"], - "attributesToHighlight": ["name"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_7() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","gender","email"], - "attributesToCrop": ["*","email:6"], - "attributesToHighlight": ["*"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_8() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender","address"], - "attributesToCrop": ["*","email:6"], - "attributesToHighlight": ["*","address"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn test_faceted_search_valid() { - // set facetting attributes before adding documents - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - - let body = json!({ - "attributesForFaceting": ["color"] - }); - server.update_all_settings(body).await; - - let dataset = include_bytes!("assets/test_set.json"); - let body: Value = serde_json::from_slice(dataset).unwrap(); - server.add_or_update_multiple_documents(body).await; - - // simple tests on attributes with string value - - let query = json!({ - "q": "a", - "facetFilters": ["color:green"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "Green")); - }); - - let query = json!({ - "q": "a", - "facetFilters": [["color:blue"]] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - let query = json!({ - "q": "a", - "facetFilters": ["color:Blue"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - // test on arrays: ["tags:bug"] - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "q": "a", - "facetFilters": ["tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); - }); - - // test and: ["color:blue", "tags:bug"] - let query = json!({ - "q": "a", - "facetFilters": ["color:blue", "tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("color") - .unwrap() == "blue" - && value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); - }); - - // test or: [["color:blue", "color:green"]] - let query = json!({ - "q": "a", - "facetFilters": [["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| - value - .get("color") - .unwrap() == "blue" - || value - .get("color") - .unwrap() == "Green")); - }); - // test and-or: ["tags:bug", ["color:blue", "color:green"]] - let query = json!({ - "q": "a", - "facetFilters": ["tags:bug", ["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| - value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())) - && (value - .get("color") - .unwrap() == "blue" - || value - .get("color") - .unwrap() == "Green"))); - - }); -} - -#[actix_rt::test] -async fn test_faceted_search_invalid() { - let mut server = common::Server::test_server().await; - - //no faceted attributes set - let query = json!({ - "q": "a", - "facetFilters": ["color:blue"] - }); - - test_post_get_search!(server, query, |response, status_code| { - - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // empty arrays are error - // [] - let query = json!({ - "q": "a", - "facetFilters": [] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - // [[]] - let query = json!({ - "q": "a", - "facetFilters": [[]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // ["color:green", []] - let query = json!({ - "q": "a", - "facetFilters": ["color:green", []] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // too much depth - // [[[]]] - let query = json!({ - "q": "a", - "facetFilters": [[[]]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // [["color:green", ["color:blue"]]] - let query = json!({ - "q": "a", - "facetFilters": [["color:green", ["color:blue"]]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // "color:green" - let query = json!({ - "q": "a", - "facetFilters": "color:green" - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); -} - -#[actix_rt::test] -async fn test_facet_count() { - let mut server = common::Server::test_server().await; - - // test without facet distribution - let query = json!({ - "q": "a", - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert!(response.get("exhaustiveFacetsCount").is_none()); - assert!(response.get("facetsDistribution").is_none()); - }); - - // test no facets set, search on color - let query = json!({ - "q": "a", - "facetsDistribution": ["color"] - }); - test_post_get_search!(server, query.clone(), |_response, status_code|{ - assert_eq!(status_code, 400); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // same as before, but now facets are set: - test_post_get_search!(server, query, |response, _status_code|{ - println!("{}", response); - assert!(response.get("exhaustiveFacetsCount").is_some()); - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 1); - // assert that case is preserved - assert!(response["facetsDistribution"] - .as_object() - .unwrap()["color"] - .as_object() - .unwrap() - .get("Green") - .is_some()); - }); - // searching on color and tags - let query = json!({ - "q": "a", - "facetsDistribution": ["color", "tags"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - let facets = response.get("facetsDistribution").unwrap().as_object().unwrap(); - assert_eq!(facets.values().count(), 2); - assert_ne!(!facets.get("color").unwrap().as_object().unwrap().values().count(), 0); - assert_ne!(!facets.get("tags").unwrap().as_object().unwrap().values().count(), 0); - }); - // wildcard - let query = json!({ - "q": "a", - "facetsDistribution": ["*"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); - }); - // wildcard with other attributes: - let query = json!({ - "q": "a", - "facetsDistribution": ["color", "*"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); - }); - - // empty facet list - let query = json!({ - "q": "a", - "facetsDistribution": [] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 0); - }); - - // attr not set as facet passed: - let query = json!({ - "q": "a", - "facetsDistribution": ["gender"] - }); - test_post_get_search!(server, query, |_response, status_code|{ - assert_eq!(status_code, 400); - }); - -} - -#[actix_rt::test] -#[should_panic] -async fn test_bad_facet_distribution() { - let mut server = common::Server::test_server().await; - // string instead of array: - let query = json!({ - "q": "a", - "facetsDistribution": "color" - }); - test_post_get_search!(server, query, |_response, _status_code| {}); - - // invalid value in array: - let query = json!({ - "q": "a", - "facetsDistribution": ["color", true] - }); - test_post_get_search!(server, query, |_response, _status_code| {}); -} - -#[actix_rt::test] -async fn highlight_cropped_text() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let doc = json!([ - { - "id": 1, - "body": r##"well, it may not work like that, try the following: -1. insert your trip -2. google your `searchQuery` -3. find a solution -> say hello"## - } - ]); - server.add_or_replace_multiple_documents(doc).await; - - // tests from #680 - //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=30"; - let query = json!({ - "q": "insert", - "attributesToHighlight": ["*"], - "attributesToCrop": ["body"], - "cropLength": 30, - }); - let expected_response = "that, try the following: \n1. insert your trip\n2. google your"; - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .get(0) - .unwrap() - .as_object() - .unwrap() - .get("_formatted") - .unwrap() - .as_object() - .unwrap() - .get("body") - .unwrap() - , &Value::String(expected_response.to_owned())); - }); - - //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=80"; - let query = json!({ - "q": "insert", - "attributesToHighlight": ["*"], - "attributesToCrop": ["body"], - "cropLength": 80, - }); - let expected_response = "well, it may not work like that, try the following: \n1. insert your trip\n2. google your `searchQuery`\n3. find a solution \n> say hello"; - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .get(0) - .unwrap() - .as_object() - .unwrap() - .get("_formatted") - .unwrap() - .as_object() - .unwrap() - .get("body") - .unwrap() - , &Value::String(expected_response.to_owned())); - }); -} - -#[actix_rt::test] -async fn well_formated_error_with_bad_request_params() { - let mut server = common::Server::with_uid("test"); - let query = "foo=bar"; - let (response, _status_code) = server.search_get(query).await; - assert!(response.get("message").is_some()); - assert!(response.get("errorCode").is_some()); - assert!(response.get("errorType").is_some()); - assert!(response.get("errorLink").is_some()); -} - - -#[actix_rt::test] -async fn update_documents_with_facet_distribution() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let settings = json!({ - "attributesForFaceting": ["genre"], - "displayedAttributes": ["genre"], - "searchableAttributes": ["genre"] - }); - server.update_all_settings(settings).await; - let update1 = json!([ - { - "id": "1", - "type": "album", - "title": "Nevermind", - "genre": ["grunge", "alternative"] - }, - { - "id": "2", - "type": "album", - "title": "Mellon Collie and the Infinite Sadness", - "genre": ["alternative", "rock"] - }, - { - "id": "3", - "type": "album", - "title": "The Queen Is Dead", - "genre": ["indie", "rock"] - } - ]); - server.add_or_update_multiple_documents(update1).await; - let search = json!({ - "q": "album", - "facetsDistribution": ["genre"] - }); - let (response1, _) = server.search_post(search.clone()).await; - let expected_facet_distribution = json!({ - "genre": { - "grunge": 1, - "alternative": 2, - "rock": 2, - "indie": 1 - } - }); - assert_json_eq!(expected_facet_distribution.clone(), response1["facetsDistribution"].clone()); - - let update2 = json!([ - { - "id": "3", - "title": "The Queen Is Very Dead" - } - ]); - server.add_or_update_multiple_documents(update2).await; - let (response2, _) = server.search_post(search).await; - assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone()); -} - -#[actix_rt::test] -async fn test_filter_nb_hits_search_normal() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let documents = json!([ - { - "id": 1, - "content": "a", - "color": "green", - "size": 1, - }, - { - "id": 2, - "content": "a", - "color": "green", - "size": 2, - }, - { - "id": 3, - "content": "a", - "color": "blue", - "size": 3, - }, - ]); - - server.add_or_update_multiple_documents(documents).await; - let (response, _) = server.search_post(json!({"q": "a"})).await; - assert_eq!(response["nbHits"], 3); - - let (response, _) = server.search_post(json!({"q": "a", "filters": "size = 1"})).await; - assert_eq!(response["nbHits"], 1); - - server.update_distinct_attribute(json!("color")).await; - - let (response, _) = server.search_post(json!({"q": "a"})).await; - assert_eq!(response["nbHits"], 2); - - let (response, _) = server.search_post(json!({"q": "a", "filters": "size < 3"})).await; - println!("result: {}", response); - assert_eq!(response["nbHits"], 1); -} diff --git a/meilisearch-http/_tests/search_settings.rs b/meilisearch-http/_tests/search_settings.rs deleted file mode 100644 index 46417498d..000000000 --- a/meilisearch-http/_tests/search_settings.rs +++ /dev/null @@ -1,538 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; -use std::convert::Into; - -mod common; - -#[actix_rt::test] -async fn search_with_settings_basic() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=ea%20exercitation&limit=3"; - - let expect = json!([ - { - "balance": "$2,467.47", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700" - }, - { - "balance": "$3,344.40", - "age": 35, - "color": "blue", - "name": "Adeline Flynn", - "gender": "female", - "email": "adelineflynn@chorizon.com", - "phone": "+1 (994) 600-2840", - "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948" - }, - { - "balance": "$3,394.96", - "age": 25, - "color": "blue", - "name": "Aida Kirby", - "gender": "female", - "email": "aidakirby@chorizon.com", - "phone": "+1 (942) 532-2325", - "address": "797 Engert Avenue, Wilsonia, Idaho, 6532" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_stop_words() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": ["ea"], - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=ea%20exercitation&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_synonyms() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "application": [ - "exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=application&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_ranking_rules() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exarcitation&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - println!("{}", response["hits"].clone()); - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_searchable_attributes() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "exarcitation": [ - "exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=Carol&limit=3"; - let expect = json!([ - { - "balance": "$1,440.09", - "age": 40, - "color": "blue", - "name": "Levy Whitley", - "gender": "male", - "email": "levywhitley@chorizon.com", - "phone": "+1 (911) 458-2411", - "address": "187 Thomas Street, Hachita, North Carolina, 2989" - }, - { - "balance": "$1,977.66", - "age": 36, - "color": "brown", - "name": "Combs Stanley", - "gender": "male", - "email": "combsstanley@chorizon.com", - "phone": "+1 (827) 419-2053", - "address": "153 Beverley Road, Siglerville, South Carolina, 3666" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_displayed_attributes() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exercitation&limit=3"; - let expect = json!([ - { - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243" - }, - { - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174" - }, - { - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_searchable_attributes_2() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exercitation&limit=3"; - let expect = json!([ - { - "age": 31, - "name": "Harper Carson", - "gender": "male" - }, - { - "age": 27, - "name": "Cherry Orr", - "gender": "female" - }, - { - "age": 28, - "name": "Maureen Dale", - "gender": "female" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -// issue #798 -#[actix_rt::test] -async fn distinct_attributes_returns_name_not_id() { - let mut server = common::Server::test_server().await; - let settings = json!({ - "distinctAttribute": "color", - }); - server.update_all_settings(settings).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["distinctAttribute"], "color"); - let (response, _) = server.get_distinct_attribute().await; - assert_eq!(response, "color"); -} diff --git a/meilisearch-http/_tests/settings.rs b/meilisearch-http/_tests/settings.rs deleted file mode 100644 index 6b125c13a..000000000 --- a/meilisearch-http/_tests/settings.rs +++ /dev/null @@ -1,523 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; -use std::convert::Into; -mod common; - -#[actix_rt::test] -async fn write_all_and_delete() { - let mut server = common::Server::test_server().await; - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all settings - - server.delete_all_settings().await; - - // 5 - Get all settings and check if they are set to default values - - let (response, _status_code) = server.get_all_settings().await; - - let expect = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - assert_json_eq!(expect, response, ordered: false); -} - -#[actix_rt::test] -async fn write_all_and_update() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Update all settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "color", - "age", - ], - "displayedAttributes": [ - "name", - "color", - "age", - "registered", - "picture", - ], - "stopWords": [], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["title"], - }); - - server.update_all_settings(body).await; - - // 5 - Get all settings and check if the content is the same of (4) - - let (response, _status_code) = server.get_all_settings().await; - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "color", - "age", - ], - "displayedAttributes": [ - "name", - "color", - "age", - "registered", - "picture", - ], - "stopWords": [], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["title"], - }); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn test_default_settings() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - - // 1 - Get all settings and compare to the previous one - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); -} - -#[actix_rt::test] -async fn test_default_settings_2() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - // 1 - Get all settings and compare to the previous one - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/516 -#[actix_rt::test] -async fn write_setting_and_update_partial() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - - // 2 - Send the settings - - let body = json!({ - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ] - }); - - server.update_all_settings(body.clone()).await; - - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - "desc(registered)", - ], - "distinctAttribute": "id", - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - }); - - server.update_all_settings(body.clone()).await; - - // 2 - Send the settings - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - "desc(registered)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn attributes_for_faceting_settings() { - let mut server = common::Server::test_server().await; - // initial attributes array should be empty - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!([])); - // add an attribute and test for its presence - let (_response, _status_code) = server.post_request_async( - "/indexes/test/settings/attributes-for-faceting", - json!(["foobar"])).await; - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!(["foobar"])); - // remove all attributes and test for emptiness - let (_response, _status_code) = server.delete_request_async( - "/indexes/test/settings/attributes-for-faceting").await; - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!([])); -} - -#[actix_rt::test] -async fn setting_ranking_rules_dont_mess_with_other_settings() { - let mut server = common::Server::test_server().await; - let body = json!({ - "rankingRules": ["asc(foobar)"] - }); - server.update_all_settings(body).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["rankingRules"].as_array().unwrap().len(), 1); - assert_eq!(response["rankingRules"].as_array().unwrap().first().unwrap().as_str().unwrap(), "asc(foobar)"); - assert!(!response["searchableAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); - assert!(!response["displayedAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); -} - -#[actix_rt::test] -async fn displayed_and_searchable_attributes_reset_to_wildcard() { - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); - - server.delete_searchable_attributes().await; - server.delete_displayed_attributes().await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); - - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); - - server.update_all_settings(json!({ "searchableAttributes": [], "displayedAttributes": [] })).await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); -} - -#[actix_rt::test] -async fn settings_that_contains_wildcard_is_wildcard() { - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color", "*"], "displayedAttributes": ["color", "*"] })).await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); -} - -#[actix_rt::test] -async fn test_displayed_attributes_field() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "age", - "email", - "gender", - "name", - "registered", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["avenue", "street"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: true); -} \ No newline at end of file diff --git a/meilisearch-http/_tests/settings_ranking_rules.rs b/meilisearch-http/_tests/settings_ranking_rules.rs deleted file mode 100644 index ac9a1e00c..000000000 --- a/meilisearch-http/_tests/settings_ranking_rules.rs +++ /dev/null @@ -1,182 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn write_all_and_delete() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_ranking_rules().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all settings - - server.delete_ranking_rules().await; - - // 5 - Get all settings and check if they are empty - - let (response, _status_code) = server.get_ranking_rules().await; - - let expected = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ]); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn write_all_and_update() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_ranking_rules().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Update all settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - ]); - - server.update_ranking_rules(body).await; - - // 5 - Get all settings and check if the content is the same of (4) - - let (response, _status_code) = server.get_ranking_rules().await; - - let expected = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - ]); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn send_undefined_rule() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let body = json!(["typos",]); - - let (_response, status_code) = server.update_ranking_rules_sync(body).await; - assert_eq!(status_code, 400); -} - -#[actix_rt::test] -async fn send_malformed_custom_rule() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let body = json!(["dsc(truc)",]); - - let (_response, status_code) = server.update_ranking_rules_sync(body).await; - assert_eq!(status_code, 400); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/521 -#[actix_rt::test] -async fn write_custom_ranking_and_index_documents() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - // 1 - Add ranking rules with one custom ranking on a string - - let body = json!(["asc(name)", "typo"]); - - server.update_ranking_rules(body).await; - - // 2 - Add documents - - let body = json!([ - { - "id": 1, - "name": "Cherry Orr", - "color": "green" - }, - { - "id": 2, - "name": "Lucas Hess", - "color": "yellow" - } - ]); - - server.add_or_replace_multiple_documents(body).await; - - // 3 - Get the first document and compare - - let expected = json!({ - "id": 1, - "name": "Cherry Orr", - "color": "green" - }); - - let (response, status_code) = server.get_document(1).await; - assert_eq!(status_code, 200); - - assert_json_eq!(response, expected, ordered: false); -} diff --git a/meilisearch-http/_tests/settings_stop_words.rs b/meilisearch-http/_tests/settings_stop_words.rs deleted file mode 100644 index 3ff2e8bb7..000000000 --- a/meilisearch-http/_tests/settings_stop_words.rs +++ /dev/null @@ -1,61 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn update_stop_words() { - let mut server = common::Server::test_server().await; - - // 1 - Get stop words - - let (response, _status_code) = server.get_stop_words().await; - assert_eq!(response.as_array().unwrap().is_empty(), true); - - // 2 - Update stop words - - let body = json!(["ut", "ea"]); - server.update_stop_words(body.clone()).await; - - // 3 - Get all stop words and compare to the previous one - - let (response, _status_code) = server.get_stop_words().await; - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all stop words - - server.delete_stop_words().await; - - // 5 - Get all stop words and check if they are empty - - let (response, _status_code) = server.get_stop_words().await; - assert_eq!(response.as_array().unwrap().is_empty(), true); -} - -#[actix_rt::test] -async fn add_documents_and_stop_words() { - let mut server = common::Server::test_server().await; - - // 2 - Update stop words - - let body = json!(["ad", "in"]); - server.update_stop_words(body.clone()).await; - - // 3 - Search for a document with stop words - - let (response, _status_code) = server.search_get("q=in%20exercitation").await; - assert!(!response["hits"].as_array().unwrap().is_empty()); - - // 4 - Search for documents with *only* stop words - - let (response, _status_code) = server.search_get("q=ad%20in").await; - assert!(response["hits"].as_array().unwrap().is_empty()); - - // 5 - Delete all stop words - - // server.delete_stop_words(); - - // // 6 - Search for a document with one stop word - - // assert!(!response["hits"].as_array().unwrap().is_empty()); -} diff --git a/meilisearch-http/_tests/url_normalizer.rs b/meilisearch-http/_tests/url_normalizer.rs deleted file mode 100644 index c2c9187ee..000000000 --- a/meilisearch-http/_tests/url_normalizer.rs +++ /dev/null @@ -1,18 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn url_normalizer() { - let mut server = common::Server::with_uid("movies"); - - let (_response, status_code) = server.get_request("/version").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("//version").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("/version/").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("//version/").await; - assert_eq!(status_code, 200); -} From c2fdb0ad4df5209deb689fe554a3d81c16cf14e0 Mon Sep 17 00:00:00 2001 From: marin Date: Mon, 1 Mar 2021 19:59:54 +0100 Subject: [PATCH 101/522] Update .github/workflows/create_artifacts.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clémentine Urquizar --- .github/workflows/create_artifacts.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/create_artifacts.yml b/.github/workflows/create_artifacts.yml index bb4546c06..21542edf0 100644 --- a/.github/workflows/create_artifacts.yml +++ b/.github/workflows/create_artifacts.yml @@ -14,14 +14,14 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] include: - os: ubuntu-latest - artifact_name: test-ci - asset_name: meilisearch-linux-amd64 + artifact_name: meilisearch-alpha + asset_name: meilisearch-alpha-linux-amd64 - os: macos-latest - artifact_name: test-ci - asset_name: meilisearch-macos-amd64 + artifact_name: meilisearch-alpha + asset_name: meilisearch-alpha-macos-amd64 - os: windows-latest - artifact_name: test-ci.exe - asset_name: meilisearch-windows-amd64.exe + artifact_name: meilisearch-alpha.exe + asset_name: meilisearch-alpha-windows-amd64.exe steps: - uses: hecrj/setup-rust-action@master with: From fc351b54d94c5a5e60a842f51e5237a16baff548 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Mar 2021 20:09:13 +0100 Subject: [PATCH 102/522] change milli revision --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9af2e7d12..17575cd48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1725,7 +1725,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?rev=8dcb3e0#8dcb3e0c41965c96ae718ae85c45004cf94c6e94" +source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" dependencies = [ "anyhow", "bstr", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 35ffb7771..b760a3d27 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -38,7 +38,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "8dcb3e0" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" From 4cf66831d46e6eeb9de5f092719696d1020414d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 2 Mar 2021 11:38:39 +0100 Subject: [PATCH 103/522] Update publish_to_docker.yml --- .github/workflows/publish_to_docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish_to_docker.yml b/.github/workflows/publish_to_docker.yml index 92a8bf5db..d724f7253 100644 --- a/.github/workflows/publish_to_docker.yml +++ b/.github/workflows/publish_to_docker.yml @@ -10,6 +10,7 @@ jobs: name: Publishing to dockerhub runs-on: ubuntu-latest steps: + - uses: actions/checkout@v1 - name: Publish to Registry uses: elgohr/Publish-Docker-Github-Action@master with: From cf97b9ff2b81562f0308ef48376e58a1ee5d68b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 2 Mar 2021 12:06:38 +0100 Subject: [PATCH 104/522] Update create_artifacts.yml --- .github/workflows/create_artifacts.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create_artifacts.yml b/.github/workflows/create_artifacts.yml index 21542edf0..94378ba83 100644 --- a/.github/workflows/create_artifacts.yml +++ b/.github/workflows/create_artifacts.yml @@ -14,13 +14,13 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] include: - os: ubuntu-latest - artifact_name: meilisearch-alpha + artifact_name: meilisearch asset_name: meilisearch-alpha-linux-amd64 - os: macos-latest - artifact_name: meilisearch-alpha + artifact_name: meilisearch asset_name: meilisearch-alpha-macos-amd64 - os: windows-latest - artifact_name: meilisearch-alpha.exe + artifact_name: meilisearch.exe asset_name: meilisearch-alpha-windows-amd64.exe steps: - uses: hecrj/setup-rust-action@master From 62532b8f79c54f9b318fbed777387173ad43756f Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Mar 2021 15:45:35 +0100 Subject: [PATCH 105/522] WIP concurent index store --- Cargo.lock | 125 ++++++++++++------ Cargo.toml | 1 + src/data/updates.rs | 5 +- .../actor_index_controller/index_actor.rs | 21 +-- .../actor_index_controller/update_actor.rs | 1 - .../actor_index_controller/update_store.rs | 6 +- 6 files changed, 106 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da8d649dd..7f3b20e89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -421,9 +421,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "assert-json-diff" @@ -447,6 +447,27 @@ dependencies = [ "tokio 0.2.24", ] +[[package]] +name = "async-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.42" @@ -609,9 +630,9 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" dependencies = [ "lazy_static", "memchr", @@ -641,10 +662,16 @@ dependencies = [ ] [[package]] -name = "byteorder" -version = "1.3.4" +name = "bytemuck" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "bytes" @@ -1022,15 +1049,15 @@ checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "winapi 0.3.9", ] [[package]] name = "flate2" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" dependencies = [ "cfg-if 1.0.0", "crc32fast", @@ -1665,9 +1692,9 @@ checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" [[package]] name = "linked-hash-map" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lmdb-rkv-sys" @@ -1691,11 +1718,11 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -1741,6 +1768,7 @@ dependencies = [ "anyhow", "assert-json-diff", "async-compression", + "async-stream", "async-trait", "byte-unit", "bytes 0.6.0", @@ -1853,7 +1881,7 @@ dependencies = [ "grenad", "heed", "human_format", - "itertools 0.9.0", + "itertools 0.10.0", "levenshtein_automata", "linked-hash-map", "log", @@ -2060,9 +2088,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "ordered-float" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dacdec97876ef3ede8c50efc429220641a0b11ba0048b4b0c357bccbc47c5204" +checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" dependencies = [ "num-traits", ] @@ -2097,7 +2125,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "smallvec", "winapi 0.3.9", ] @@ -2496,10 +2524,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] -name = "regex" -version = "1.4.2" +name = "redox_syscall" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "aho-corasick", "memchr", @@ -2518,9 +2555,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "remove_dir_all" @@ -2578,6 +2615,12 @@ dependencies = [ "quick-error", ] +[[package]] +name = "retain_mut" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" + [[package]] name = "ring" version = "0.16.19" @@ -2595,11 +2638,13 @@ dependencies = [ [[package]] name = "roaring" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d60b41c8f25d07cecab125cb46ebbf234fc055effc61ca2392a3ef4f9422304" +checksum = "c6744a4a918e91359ad1d356a91e2e943a86d9fb9ae77f715d617032ea2af88f" dependencies = [ + "bytemuck", "byteorder", + "retain_mut", ] [[package]] @@ -2721,18 +2766,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.118" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.118" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" dependencies = [ "proc-macro2", "quote", @@ -2741,9 +2786,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.60" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "indexmap", "itoa", @@ -2983,9 +3028,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.55" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a571a711dddd09019ccc628e1b17fe87c59b09d513c06c026877aa708334f37a" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" dependencies = [ "proc-macro2", "quote", @@ -3021,7 +3066,7 @@ checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" dependencies = [ "filetime", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "xattr", ] @@ -3037,14 +3082,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "rand 0.7.3", - "redox_syscall", + "rand 0.8.3", + "redox_syscall 0.2.5", "remove_dir_all", "winapi 0.3.9", ] diff --git a/Cargo.toml b/Cargo.toml index 126f859e7..aaf571ef3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ itertools = "0.10.0" either = "1.6.1" async-trait = "0.1.42" thiserror = "1.0.24" +async-stream = "0.3.0" [dependencies.sentry] default-features = false diff --git a/src/data/updates.rs b/src/data/updates.rs index a559d812a..7db04cd5d 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -23,10 +23,11 @@ impl Data { { let file = tempfile::tempfile_in(".")?; let mut file = File::from_std(file); - while let Some(Ok(bytes)) = stream.next().await { - file.write(bytes.as_ref()).await; + while let Some(item) = stream.next().await { + file.write_all(&item?).await?; } file.seek(std::io::SeekFrom::Start(0)).await?; + file.flush().await?; let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, file, primary_key).await?; Ok(update_status) } diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 0cb057a9b..7100edef6 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -13,7 +13,7 @@ use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use log::info; use crate::data::SearchQuery; -use futures::stream::{StreamExt, Stream}; +use futures::stream::StreamExt; use super::update_handler::UpdateHandler; use async_stream::stream; @@ -32,7 +32,7 @@ enum IndexMsg { } struct IndexActor { - inbox: mpsc::Receiver, + inbox: Option>, update_handler: Arc, store: S, } @@ -57,26 +57,31 @@ impl IndexActor { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).unwrap(); let update_handler = Arc::new(update_handler); + let inbox = Some(inbox); Self { inbox, store, update_handler } } async fn run(mut self) { + let mut inbox = self.inbox.take().expect("Index Actor must have a inbox at this point."); + let stream = stream! { loop { - match self.inbox.recv().await { + match inbox.recv().await { Some(msg) => yield msg, None => break, } } }; - stream.for_each_concurent(Some(10), |msg| { + let fut = stream.for_each_concurrent(Some(10), |msg| async { match msg { - IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret), - IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret), - IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret), + IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret).await, + IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret).await, + IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, } - }) + }); + + fut.await; } async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index d182ef1c8..3d783009f 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -41,7 +41,6 @@ impl UpdateActor { } async fn run(mut self) { - info!("started update actor."); loop { diff --git a/src/index_controller/actor_index_controller/update_store.rs b/src/index_controller/actor_index_controller/update_store.rs index 371ac7bd9..d14f47f05 100644 --- a/src/index_controller/actor_index_controller/update_store.rs +++ b/src/index_controller/actor_index_controller/update_store.rs @@ -1,6 +1,6 @@ use std::path::Path; use std::sync::{Arc, RwLock}; -use std::io::{Cursor, SeekFrom, Seek}; +use std::io::{Cursor, SeekFrom, Seek, Write}; use crossbeam_channel::Sender; use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; @@ -192,7 +192,9 @@ where .replace(processing.clone()); let mut cursor = Cursor::new(first_content); let mut file = tempfile::tempfile()?; - std::io::copy(&mut cursor, &mut file)?; + let n = std::io::copy(&mut cursor, &mut file)?; + println!("copied count: {}", n); + file.flush()?; file.seek(SeekFrom::Start(0))?; // Process the pending update using the provided user function. let result = handler.handle_update(processing, file); From 9aca6fab88857dec3ce55b3bc424446d35bb7834 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 10:57:13 +0100 Subject: [PATCH 106/522] completely file backed udpates --- src/data/mod.rs | 2 +- src/data/updates.rs | 14 +--- .../actor_index_controller/mod.rs | 28 +++++-- .../actor_index_controller/update_actor.rs | 84 ++++++++++++------- .../actor_index_controller/update_store.rs | 22 ++--- src/index_controller/mod.rs | 4 +- 6 files changed, 90 insertions(+), 64 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index ecc8a7e2e..0b57f9a55 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -62,7 +62,7 @@ impl Data { let path = options.db_path.clone(); //let indexer_opts = options.indexer_options.clone(); create_dir_all(&path)?; - let index_controller = ActorIndexController::new(); + let index_controller = ActorIndexController::new(&path); let index_controller = Arc::new(index_controller); let mut api_keys = ApiKeys { diff --git a/src/data/updates.rs b/src/data/updates.rs index 7db04cd5d..01f5174a2 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -3,9 +3,6 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; //use tokio::io::AsyncWriteExt; use actix_web::web::Payload; -use tokio::fs::File; -use tokio::io::{AsyncWriteExt, AsyncSeekExt}; -use futures::prelude::stream::StreamExt; use crate::index_controller::UpdateStatus; use crate::index_controller::{Settings, IndexMetadata}; @@ -17,18 +14,11 @@ impl Data { index: impl AsRef + Send + Sync + 'static, method: IndexDocumentsMethod, format: UpdateFormat, - mut stream: Payload, + stream: Payload, primary_key: Option, ) -> anyhow::Result { - let file = tempfile::tempfile_in(".")?; - let mut file = File::from_std(file); - while let Some(item) = stream.next().await { - file.write_all(&item?).await?; - } - file.seek(std::io::SeekFrom::Start(0)).await?; - file.flush().await?; - let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, file, primary_key).await?; + let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, stream, primary_key).await?; Ok(update_status) } diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 646d9cf45..58e58a45a 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -4,25 +4,29 @@ mod uuid_resolver; mod update_store; mod update_handler; -use tokio::sync::oneshot; +use std::path::Path; + +use tokio::sync::{mpsc, oneshot}; use super::IndexController; use uuid::Uuid; use super::IndexMetadata; -use tokio::fs::File; +use futures::stream::StreamExt; +use actix_web::web::Payload; use super::UpdateMeta; use crate::data::{SearchResult, SearchQuery}; +use actix_web::web::Bytes; pub struct ActorIndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, - update_handle: update_actor::UpdateActorHandle, + update_handle: update_actor::UpdateActorHandle, } impl ActorIndexController { - pub fn new() -> Self { + pub fn new(path: impl AsRef) -> Self { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); let index_actor = index_actor::IndexActorHandle::new(); - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone()); + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); Self { uuid_resolver, index_handle: index_actor, update_handle } } } @@ -43,12 +47,22 @@ impl IndexController for ActorIndexController { index: String, method: milli::update::IndexDocumentsMethod, format: milli::update::UpdateFormat, - data: File, + mut payload: Payload, primary_key: Option, ) -> anyhow::Result { let uuid = self.uuid_resolver.get_or_create(index).await?; let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; - let status = self.update_handle.update(meta, Some(data), uuid).await?; + let (sender, receiver) = mpsc::channel(10); + + // It is necessary to spawn a local task to senf the payload to the update handle to + // prevent dead_locking between the update_handle::update that waits for the update to be + // registered and the update_actor that waits for the the payload to be sent to it. + tokio::task::spawn_local(async move { + while let Some(bytes) = payload.next().await { + sender.send(bytes.unwrap()).await; + } + }); + let status = self.update_handle.update(meta, receiver, uuid).await?; Ok(status) } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index 3d783009f..6fc873715 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -1,22 +1,24 @@ -use super::index_actor::IndexActorHandle; -use uuid::Uuid; -use tokio::sync::{mpsc, oneshot}; -use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult}; -use thiserror::Error; -use tokio::io::AsyncReadExt; -use log::info; -use tokio::fs::File; -use std::path::PathBuf; use std::fs::create_dir_all; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use log::info; +use super::index_actor::IndexActorHandle; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; + +use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult, updates::Pending}; + pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; #[derive(Debug, Error)] pub enum UpdateError {} -enum UpdateMsg { +enum UpdateMsg { CreateIndex{ uuid: Uuid, ret: oneshot::Sender>, @@ -24,20 +26,30 @@ enum UpdateMsg { Update { uuid: Uuid, meta: UpdateMeta, - payload: Option, + data: mpsc::Receiver, ret: oneshot::Sender> } } -struct UpdateActor { +struct UpdateActor { + path: PathBuf, store: Arc, - inbox: mpsc::Receiver, + inbox: mpsc::Receiver>, index_handle: IndexActorHandle, } -impl UpdateActor { - fn new(store: Arc, inbox: mpsc::Receiver, index_handle: IndexActorHandle) -> Self { - Self { store, inbox, index_handle } +impl UpdateActor +where D: AsRef<[u8]> + Sized + 'static, +{ + fn new( + store: Arc, + inbox: mpsc::Receiver>, + index_handle: IndexActorHandle, + path: impl AsRef, + ) -> Self { + let path = path.as_ref().to_owned().join("update_files"); + create_dir_all(&path).unwrap(); + Self { store, inbox, index_handle, path } } async fn run(mut self) { @@ -45,29 +57,43 @@ impl UpdateActor { loop { match self.inbox.recv().await { - Some(UpdateMsg::Update { uuid, meta, payload, ret }) => self.handle_update(uuid, meta, payload, ret).await, + Some(UpdateMsg::Update { uuid, meta, data, ret }) => self.handle_update(uuid, meta, data, ret).await, Some(_) => {} None => {} } } } - async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, payload: Option, ret: oneshot::Sender>) { - let mut buf = Vec::new(); - let mut payload = payload.unwrap(); - payload.read_to_end(&mut buf).await.unwrap(); - let result = self.store.register_update(meta, &buf, uuid).unwrap(); + async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver, ret: oneshot::Sender>) { + let store = self.store.clone(); + let update_file_id = uuid::Uuid::new_v4(); + let path = self.path.join(format!("update_{}", update_file_id)); + let mut file = File::create(&path).await.unwrap(); + + while let Some(bytes) = payload.recv().await { + file.write_all(bytes.as_ref()).await; + } + + file.flush().await; + + let file = file.into_std().await; + + let result = tokio::task::spawn_blocking(move || -> anyhow::Result> { + Ok(store.register_update(meta, path, uuid)?) + }).await.unwrap().unwrap(); let _ = ret.send(Ok(UpdateStatus::Pending(result))); } } #[derive(Clone)] -pub struct UpdateActorHandle { - sender: mpsc::Sender, +pub struct UpdateActorHandle { + sender: mpsc::Sender>, } -impl UpdateActorHandle { - pub fn new(index_handle: IndexActorHandle) -> Self { +impl UpdateActorHandle +where D: AsRef<[u8]> + Sized + 'static, +{ + pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { let (sender, receiver) = mpsc::channel(100); let mut options = heed::EnvOpenOptions::new(); options.map_size(4096 * 100_000); @@ -79,16 +105,16 @@ impl UpdateActorHandle { let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle_clone.update(meta, file)) }).unwrap(); - let actor = UpdateActor::new(store, receiver, index_handle); + let actor = UpdateActor::new(store, receiver, index_handle, path); tokio::task::spawn_local(actor.run()); Self { sender } } - pub async fn update(&self, meta: UpdateMeta, payload: Option, uuid: Uuid) -> Result { + pub async fn update(&self, meta: UpdateMeta, data: mpsc::Receiver, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Update { uuid, - payload, + data, meta, ret, }; diff --git a/src/index_controller/actor_index_controller/update_store.rs b/src/index_controller/actor_index_controller/update_store.rs index d14f47f05..ae4bfb8d8 100644 --- a/src/index_controller/actor_index_controller/update_store.rs +++ b/src/index_controller/actor_index_controller/update_store.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; -use std::io::{Cursor, SeekFrom, Seek, Write}; +use std::fs::remove_file; use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson}; use heed::{EnvOpenOptions, Env, Database}; use serde::{Serialize, Deserialize}; use std::fs::File; @@ -17,7 +17,7 @@ type BEU64 = heed::zerocopy::U64; pub struct UpdateStore { env: Env, pending_meta: Database, SerdeJson>>, - pending: Database, ByteSlice>, + pending: Database, SerdeJson>, processed_meta: Database, SerdeJson>>, failed_meta: Database, SerdeJson>>, aborted_meta: Database, SerdeJson>>, @@ -140,7 +140,7 @@ where pub fn register_update( &self, meta: M, - content: &[u8], + content: impl AsRef, index_uuid: Uuid, ) -> heed::Result> { let mut wtxn = self.env.write_txn()?; @@ -154,7 +154,7 @@ where let meta = Pending::new(meta, update_id, index_uuid); self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, content)?; + self.pending.put(&mut wtxn, &update_key, &content.as_ref().to_owned())?; wtxn.commit()?; @@ -178,7 +178,7 @@ where // a reader while processing it, not a writer. match first_meta { Some((first_id, pending)) => { - let first_content = self.pending + let content_path = self.pending .get(&rtxn, &first_id)? .expect("associated update content"); @@ -190,12 +190,7 @@ where .write() .unwrap() .replace(processing.clone()); - let mut cursor = Cursor::new(first_content); - let mut file = tempfile::tempfile()?; - let n = std::io::copy(&mut cursor, &mut file)?; - println!("copied count: {}", n); - file.flush()?; - file.seek(SeekFrom::Start(0))?; + let file = File::open(&content_path)?; // Process the pending update using the provided user function. let result = handler.handle_update(processing, file); drop(rtxn); @@ -209,6 +204,7 @@ where .unwrap() .take(); self.pending_meta.delete(&mut wtxn, &first_id)?; + remove_file(&content_path)?; self.pending.delete(&mut wtxn, &first_id)?; match result { Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index cee276ea0..c3a09b2aa 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -12,7 +12,7 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; -use tokio::fs::File; +use actix_web::web::Payload; use crate::data::SearchResult; use crate::data::SearchQuery; @@ -133,7 +133,7 @@ pub trait IndexController { index: String, method: IndexDocumentsMethod, format: UpdateFormat, - data: File, + data: Payload, primary_key: Option, ) -> anyhow::Result; From 7c7143d435dee28f1772c2532f446c31c14e940a Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 11:43:51 +0100 Subject: [PATCH 107/522] remove IndexController interface --- src/data/mod.rs | 10 +- .../actor_index_controller/mod.rs | 32 ++-- src/index_controller/mod.rs | 168 ------------------ 3 files changed, 18 insertions(+), 192 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 0b57f9a55..f2798c868 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -9,8 +9,8 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexController, IndexMetadata, Settings, IndexSettings}; -use crate::index_controller::actor_index_controller::ActorIndexController; +use crate::index_controller::{IndexMetadata, Settings, IndexSettings}; +use crate::index_controller::actor_index_controller::IndexController; use crate::option::Opt; #[derive(Clone)] @@ -26,9 +26,8 @@ impl Deref for Data { } } -#[derive(Clone)] pub struct DataInner { - pub index_controller: Arc, + pub index_controller: IndexController, pub api_keys: ApiKeys, options: Opt, } @@ -62,8 +61,7 @@ impl Data { let path = options.db_path.clone(); //let indexer_opts = options.indexer_options.clone(); create_dir_all(&path)?; - let index_controller = ActorIndexController::new(&path); - let index_controller = Arc::new(index_controller); + let index_controller = IndexController::new(&path); let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 58e58a45a..34fcf55f0 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -7,7 +7,6 @@ mod update_handler; use std::path::Path; use tokio::sync::{mpsc, oneshot}; -use super::IndexController; use uuid::Uuid; use super::IndexMetadata; use futures::stream::StreamExt; @@ -16,21 +15,12 @@ use super::UpdateMeta; use crate::data::{SearchResult, SearchQuery}; use actix_web::web::Bytes; -pub struct ActorIndexController { +pub struct IndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, update_handle: update_actor::UpdateActorHandle, } -impl ActorIndexController { - pub fn new(path: impl AsRef) -> Self { - let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); - let index_actor = index_actor::IndexActorHandle::new(); - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); - Self { uuid_resolver, index_handle: index_actor, update_handle } - } -} - enum IndexControllerMsg { CreateIndex { uuid: Uuid, @@ -40,9 +30,15 @@ enum IndexControllerMsg { Shutdown, } -#[async_trait::async_trait(?Send)] -impl IndexController for ActorIndexController { - async fn add_documents( +impl IndexController { + pub fn new(path: impl AsRef) -> Self { + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); + let index_actor = index_actor::IndexActorHandle::new(); + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); + Self { uuid_resolver, index_handle: index_actor, update_handle } + } + + pub async fn add_documents( &self, index: String, method: milli::update::IndexDocumentsMethod, @@ -78,7 +74,7 @@ impl IndexController for ActorIndexController { todo!() } - async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { + pub async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { let super::IndexSettings { name, primary_key } = index_settings; let uuid = self.uuid_resolver.create(name.unwrap()).await?; let index_meta = self.index_handle.create_index(uuid, primary_key).await?; @@ -93,7 +89,7 @@ impl IndexController for ActorIndexController { todo!() } - fn index(&self, name: String) -> anyhow::Result>> { + pub fn index(&self, name: String) -> anyhow::Result>> { todo!() } @@ -105,7 +101,7 @@ impl IndexController for ActorIndexController { todo!() } - fn list_indexes(&self) -> anyhow::Result> { + pub fn list_indexes(&self) -> anyhow::Result> { todo!() } @@ -113,7 +109,7 @@ impl IndexController for ActorIndexController { todo!() } - async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { + pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); let result = self.index_handle.search(uuid, query).await?; Ok(result) diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index c3a09b2aa..fbe7a161d 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,20 +1,14 @@ pub mod actor_index_controller; -//mod local_index_controller; mod updates; use std::collections::HashMap; use std::num::NonZeroUsize; -use std::sync::Arc; use anyhow::Result; use chrono::{DateTime, Utc}; -use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; -use actix_web::web::Payload; -use crate::data::SearchResult; -use crate::data::SearchQuery; pub use updates::{Processed, Processing, Failed}; @@ -109,165 +103,3 @@ pub struct IndexSettings { pub name: Option, pub primary_key: Option, } - -/// The `IndexController` is in charge of the access to the underlying indices. It splits the logic -/// for read access which is provided thanks to an handle to the index, and write access which must -/// be provided. This allows the implementer to define the behaviour of write accesses to the -/// indices, and abstract the scheduling of the updates. The implementer must be able to provide an -/// instance of `IndexStore` -#[async_trait::async_trait(?Send)] -pub trait IndexController { - - /* - * Write operations - * - * Logic for the write operation need to be provided by the implementer, since they can be made - * asynchronous thanks to an update_store for example. - * - * */ - - /// Perform document addition on the database. If the provided index does not exist, it will be - /// created when the addition is applied to the index. - async fn add_documents( - &self, - index: String, - method: IndexDocumentsMethod, - format: UpdateFormat, - data: Payload, - primary_key: Option, - ) -> anyhow::Result; - - async fn search(&self, name: String, query: SearchQuery) -> Result; - - /// Clear all documents in the given index. - fn clear_documents(&self, index: String) -> anyhow::Result; - - /// Delete all documents in `document_ids`. - fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result; - - /// Updates an index settings. If the index does not exist, it will be created when the update - /// is applied to the index. - fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result; - - /// Create an index with the given `index_uid`. - async fn create_index(&self, index_settings: IndexSettings) -> Result; - - /// Delete index with the given `index_uid`, attempting to close it beforehand. - fn delete_index(&self, index_uid: String) -> Result<()>; - - /// Swap two indexes, concretely, it simply swaps the index the names point to. - fn swap_indices(&self, index1_uid: String, index2_uid: String) -> Result<()>; - - /// Returns, if it exists, the `Index` with the povided name. - fn index(&self, name: String) -> anyhow::Result>>; - - /// Returns the udpate status an update - fn update_status(&self, index: String, id: u64) -> anyhow::Result>; - - /// Returns all the udpate status for an index - fn all_update_status(&self, index: String) -> anyhow::Result>; - - /// List all the indexes - fn list_indexes(&self) -> anyhow::Result>; - - fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result; -} - - -#[cfg(test)] -#[macro_use] -pub(crate) mod test { - use super::*; - - #[macro_export] - macro_rules! make_index_controller_tests { - ($controller_buider:block) => { - #[test] - fn test_create_and_list_indexes() { - crate::index_controller::test::create_and_list_indexes($controller_buider); - } - - #[test] - fn test_create_index_with_no_name_is_error() { - crate::index_controller::test::create_index_with_no_name_is_error($controller_buider); - } - - #[test] - fn test_update_index() { - crate::index_controller::test::update_index($controller_buider); - } - }; - } - - pub(crate) fn create_and_list_indexes(controller: impl IndexController) { - let settings1 = IndexSettings { - name: Some(String::from("test_index")), - primary_key: None, - }; - - let settings2 = IndexSettings { - name: Some(String::from("test_index2")), - primary_key: Some(String::from("foo")), - }; - - controller.create_index(settings1).unwrap(); - controller.create_index(settings2).unwrap(); - - let indexes = controller.list_indexes().unwrap(); - assert_eq!(indexes.len(), 2); - assert_eq!(indexes[0].uid, "test_index"); - assert_eq!(indexes[1].uid, "test_index2"); - assert_eq!(indexes[1].primary_key.clone().unwrap(), "foo"); - } - - pub(crate) fn create_index_with_no_name_is_error(controller: impl IndexController) { - let settings = IndexSettings { - name: None, - primary_key: None, - }; - assert!(controller.create_index(settings).is_err()); - } - - pub(crate) fn update_index(controller: impl IndexController) { - - let settings = IndexSettings { - name: Some(String::from("test")), - primary_key: None, - }; - - assert!(controller.create_index(settings).is_ok()); - - // perform empty update returns index meta unchanged - let settings = IndexSettings { - name: None, - primary_key: None, - }; - - let result = controller.update_index("test", settings).unwrap(); - assert_eq!(result.uid, "test"); - assert_eq!(result.created_at, result.updated_at); - assert!(result.primary_key.is_none()); - - // Changing the name trigger an error - let settings = IndexSettings { - name: Some(String::from("bar")), - primary_key: None, - }; - - assert!(controller.update_index("test", settings).is_err()); - - // Update primary key - let settings = IndexSettings { - name: None, - primary_key: Some(String::from("foo")), - }; - - let result = controller.update_index("test", settings.clone()).unwrap(); - assert_eq!(result.uid, "test"); - assert!(result.created_at < result.updated_at); - assert_eq!(result.primary_key.unwrap(), "foo"); - - // setting the primary key again is an error - assert!(controller.update_index("test", settings).is_err()); - } -} From 70d935a2da2fc73db469aa9ccc73a87f6e06ade5 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 11:53:01 +0100 Subject: [PATCH 108/522] refactor index serach for better error handling --- .../actor_index_controller/index_actor.rs | 163 +++++++++--------- 1 file changed, 83 insertions(+), 80 deletions(-) diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 7100edef6..48b10c8a3 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -87,85 +87,7 @@ impl IndexActor { async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - - let before_search = Instant::now(); - let rtxn = index.read_txn().unwrap(); - - let mut search = index.search(&rtxn); - - if let Some(ref query) = query.q { - search.query(query); - } - - search.limit(query.limit); - search.offset(query.offset.unwrap_or_default()); - - //if let Some(ref facets) = query.facet_filters { - //if let Some(facets) = parse_facets(facets, index, &rtxn)? { - //search.facet_condition(facets); - //} - //} - let milli::SearchResult { - documents_ids, - found_words, - candidates, - .. - } = search.execute().unwrap(); - let mut documents = Vec::new(); - let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); - - let displayed_fields_ids = index.displayed_fields_ids(&rtxn).unwrap(); - - let attributes_to_retrieve_ids = match query.attributes_to_retrieve { - Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, - Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), - None => None, - }; - - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { - (_, Some(ids)) => ids, - (Some(ids), None) => ids, - (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let stop_words = fst::Set::default(); - let highlighter = crate::data::search::Highlighter::new(&stop_words); - - for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { - let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); - if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); - } - documents.push(object); - } - - let nb_hits = candidates.len(); - - let facet_distributions = match query.facet_distributions { - Some(ref fields) => { - let mut facet_distribution = index.facets_distribution(&rtxn); - if fields.iter().all(|f| f != "*") { - facet_distribution.facets(fields); - } - Some(facet_distribution.candidates(candidates).execute().unwrap()) - } - None => None, - }; - - let result = Ok(SearchResult { - hits: documents, - nb_hits, - query: query.q.clone().unwrap_or_default(), - limit: query.limit, - offset: query.offset.unwrap_or_default(), - processing_time_ms: before_search.elapsed().as_millis(), - facet_distributions, - }); - + let result = perform_search(&index, query); ret.send(result) }); @@ -177,7 +99,7 @@ impl IndexActor { } async fn handle_update(&self, meta: Processing, data: File, ret: oneshot::Sender) { - info!("processing update"); + info!("processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); @@ -187,6 +109,87 @@ impl IndexActor { } } +fn perform_search(index: &Index, query: SearchQuery) -> anyhow::Result { + let before_search = Instant::now(); + let rtxn = index.read_txn()?; + + let mut search = index.search(&rtxn); + + if let Some(ref query) = query.q { + search.query(query); + } + + search.limit(query.limit); + search.offset(query.offset.unwrap_or_default()); + + //if let Some(ref facets) = query.facet_filters { + //if let Some(facets) = parse_facets(facets, index, &rtxn)? { + //search.facet_condition(facets); + //} + //} + let milli::SearchResult { + documents_ids, + found_words, + candidates, + .. + } = search.execute()?; + let mut documents = Vec::new(); + let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields_ids = index.displayed_fields_ids(&rtxn).unwrap(); + + let attributes_to_retrieve_ids = match query.attributes_to_retrieve { + Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, + Some(ref attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>() + .into(), + None => None, + }; + + let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { + (_, Some(ids)) => ids, + (Some(ids), None) => ids, + (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let stop_words = fst::Set::default(); + let highlighter = crate::data::search::Highlighter::new(&stop_words); + + for (_id, obkv) in index.documents(&rtxn, documents_ids)? { + let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); + if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { + highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + } + documents.push(object); + } + + let nb_hits = candidates.len(); + + let facet_distributions = match query.facet_distributions { + Some(ref fields) => { + let mut facet_distribution = index.facets_distribution(&rtxn); + if fields.iter().all(|f| f != "*") { + facet_distribution.facets(fields); + } + Some(facet_distribution.candidates(candidates).execute()?) + } + None => None, + }; + + let result = SearchResult { + hits: documents, + nb_hits, + query: query.q.clone().unwrap_or_default(), + limit: query.limit, + offset: query.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + facet_distributions, + }; + Ok(result) +} + #[derive(Clone)] pub struct IndexActorHandle { sender: mpsc::Sender, From e285404c3e7fc5d9f490085f685d260a52096936 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 12:16:16 +0100 Subject: [PATCH 109/522] handle errors when sendign payload to actor --- .../actor_index_controller/mod.rs | 9 ++++- .../actor_index_controller/update_actor.rs | 35 +++++++++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 34fcf55f0..be63b42d4 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -55,9 +55,16 @@ impl IndexController { // registered and the update_actor that waits for the the payload to be sent to it. tokio::task::spawn_local(async move { while let Some(bytes) = payload.next().await { - sender.send(bytes.unwrap()).await; + match bytes { + Ok(bytes) => { sender.send(Ok(bytes)).await; }, + Err(e) => { + let error: Box = Box::new(e); + sender.send(Err(error)).await; }, + } } }); + + // This must be done *AFTER* spawning the task. let status = self.update_handle.update(meta, receiver, uuid).await?; Ok(status) } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index 6fc873715..1541cd4f7 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -10,13 +10,17 @@ use uuid::Uuid; use tokio::fs::File; use tokio::io::AsyncWriteExt; -use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult, updates::Pending}; +use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult}; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; +type PayloadData = std::result::Result>; #[derive(Debug, Error)] -pub enum UpdateError {} +pub enum UpdateError { + #[error("error with update: {0}")] + Error(Box), +} enum UpdateMsg { CreateIndex{ @@ -26,7 +30,7 @@ enum UpdateMsg { Update { uuid: Uuid, meta: UpdateMeta, - data: mpsc::Receiver, + data: mpsc::Receiver>, ret: oneshot::Sender> } } @@ -64,24 +68,35 @@ where D: AsRef<[u8]> + Sized + 'static, } } - async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver, ret: oneshot::Sender>) { + async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver>, ret: oneshot::Sender>) { let store = self.store.clone(); let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); let mut file = File::create(&path).await.unwrap(); while let Some(bytes) = payload.recv().await { - file.write_all(bytes.as_ref()).await; + match bytes { + Ok(bytes) => { + file.write_all(bytes.as_ref()).await; + } + Err(e) => { + ret.send(Err(UpdateError::Error(e))); + return + } + } } file.flush().await; let file = file.into_std().await; - let result = tokio::task::spawn_blocking(move || -> anyhow::Result> { - Ok(store.register_update(meta, path, uuid)?) - }).await.unwrap().unwrap(); - let _ = ret.send(Ok(UpdateStatus::Pending(result))); + let result = tokio::task::spawn_blocking(move || { + let result = store + .register_update(meta, path, uuid) + .map(|pending| UpdateStatus::Pending(pending)) + .map_err(|e| UpdateError::Error(Box::new(e))); + let _ = ret.send(result); + }).await; } } @@ -110,7 +125,7 @@ where D: AsRef<[u8]> + Sized + 'static, Self { sender } } - pub async fn update(&self, meta: UpdateMeta, data: mpsc::Receiver, uuid: Uuid) -> Result { + pub async fn update(&self, meta: UpdateMeta, data: mpsc::Receiver>, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Update { uuid, From 3cd799a74442985347720b2c16a0686667711b67 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 14:39:44 +0100 Subject: [PATCH 110/522] fix update files created in the wrong place --- src/data/mod.rs | 2 ++ .../actor_index_controller/index_actor.rs | 4 ++-- src/index_controller/actor_index_controller/mod.rs | 2 +- .../actor_index_controller/update_actor.rs | 9 ++++++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index f2798c868..aa601a670 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -59,7 +59,9 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); + //let indexer_opts = options.indexer_options.clone(); + create_dir_all(&path)?; let index_controller = IndexController::new(&path); diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 48b10c8a3..04920260e 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -196,10 +196,10 @@ pub struct IndexActorHandle { } impl IndexActorHandle { - pub fn new() -> Self { + pub fn new(path: impl AsRef) -> Self { let (sender, receiver) = mpsc::channel(100); - let store = MapIndexStore::new("data.ms"); + let store = MapIndexStore::new(path); let actor = IndexActor::new(receiver, store); tokio::task::spawn(actor.run()); Self { sender } diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index be63b42d4..882b20963 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -33,7 +33,7 @@ enum IndexControllerMsg { impl IndexController { pub fn new(path: impl AsRef) -> Self { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); - let index_actor = index_actor::IndexActorHandle::new(); + let index_actor = index_actor::IndexActorHandle::new(&path); let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); Self { uuid_resolver, index_handle: index_actor, update_handle } } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index 1541cd4f7..e82f01092 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -112,9 +112,12 @@ where D: AsRef<[u8]> + Sized + 'static, let (sender, receiver) = mpsc::channel(100); let mut options = heed::EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let mut path = PathBuf::new(); - path.push("data.ms"); - path.push("updates"); + + let path = path + .as_ref() + .to_owned() + .join("updates"); + create_dir_all(&path).unwrap(); let index_handle_clone = index_handle.clone(); let store = UpdateStore::open(options, &path, move |meta, file| { From eff8570f591fe32a6106087807e3fe8c18e8e5e4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 15:10:00 +0100 Subject: [PATCH 111/522] handle ctrl-c shutdown --- src/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index c9042b88e..c5e2d11e6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -112,7 +112,9 @@ async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box .wrap(middleware::Logger::default()) .wrap(middleware::Compress::default()) .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - }); + }) + // Disable signals allows the server to terminate immediately when a user enter CTRL-C + .disable_signals(); if let Some(config) = opt.get_ssl_config()? { http_server From ae3c8af56cbd938666b4a3b4bb23db871c5c38c5 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 10:40:37 +0100 Subject: [PATCH 112/522] enable faceted search --- .../actor_index_controller/index_actor.rs | 76 +++++++++++++++---- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 04920260e..b45b683bf 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -1,23 +1,24 @@ +use std::collections::{HashMap, hash_map::Entry}; use std::fs::{File, create_dir_all}; use std::path::{PathBuf, Path}; use std::sync::Arc; use std::time::Instant; +use anyhow::bail; +use either::Either; +use async_stream::stream; use chrono::Utc; -use heed::EnvOpenOptions; -use milli::Index; -use std::collections::HashMap; -use std::collections::hash_map::Entry; +use futures::stream::StreamExt; +use heed::{EnvOpenOptions, RoTxn}; +use log::info; +use milli::{FacetCondition, Index}; +use serde_json::Value; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; -use log::info; -use crate::data::SearchQuery; -use futures::stream::StreamExt; use super::update_handler::UpdateHandler; -use async_stream::stream; -use crate::data::SearchResult; +use crate::data::{SearchQuery, SearchResult}; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; use crate::option::IndexerOpts; @@ -99,7 +100,7 @@ impl IndexActor { } async fn handle_update(&self, meta: Processing, data: File, ret: oneshot::Sender) { - info!("processing update {}", meta.id()); + info!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); @@ -122,11 +123,12 @@ fn perform_search(index: &Index, query: SearchQuery) -> anyhow::Result anyhow::Result, +) -> anyhow::Result> { + let mut ands = Vec::new(); + for value in arr { + match value { + Value::String(s) => ands.push(Either::Right(s.clone())), + Value::Array(arr) => { + let mut ors = Vec::new(); + for value in arr { + match value { + Value::String(s) => ors.push(s.clone()), + v => bail!("Invalid facet expression, expected String, found: {:?}", v), + } + } + ands.push(Either::Left(ors)); + } + v => bail!( + "Invalid facet expression, expected String or [String], found: {:?}", + v + ), + } + } + + FacetCondition::from_array(txn, index, ands) +} + +fn parse_facets( + facets: &Value, + index: &Index, + txn: &RoTxn, +) -> anyhow::Result> { + match facets { + // Disabled for now + //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), + Value::Array(arr) => parse_facets_array(txn, index, arr), + v => bail!( + "Invalid facet expression, expected Array, found: {:?}", + v + ), + } +} #[derive(Clone)] pub struct IndexActorHandle { sender: mpsc::Sender, From 9e2a95b1a372ee2b8909807e16207fa8ed989b2f Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 11:23:41 +0100 Subject: [PATCH 113/522] refactor search --- src/data/mod.rs | 2 - src/data/search.rs | 243 +----------------- .../actor_index_controller/index_actor.rs | 154 +---------- .../actor_index_controller/mod.rs | 2 +- .../actor_index_controller/update_handler.rs | 14 +- src/lib.rs | 1 + src/routes/search.rs | 2 +- 7 files changed, 23 insertions(+), 395 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index aa601a670..58acb105a 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -1,8 +1,6 @@ pub mod search; mod updates; -pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; - use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; diff --git a/src/data/search.rs b/src/data/search.rs index 6ab792073..d6bf8438e 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -1,203 +1,8 @@ -use std::collections::{HashSet, BTreeMap}; -use std::mem; -use std::time::Instant; - -use anyhow::bail; -use either::Either; -use heed::RoTxn; -use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{obkv_to_json, FacetCondition, Index, facet::FacetValue}; -use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use crate::index::{SearchQuery, SearchResult}; use super::Data; -pub const DEFAULT_SEARCH_LIMIT: usize = 20; - -const fn default_search_limit() -> usize { - DEFAULT_SEARCH_LIMIT -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -#[allow(dead_code)] -pub struct SearchQuery { - pub q: Option, - pub offset: Option, - #[serde(default = "default_search_limit")] - pub limit: usize, - pub attributes_to_retrieve: Option>, - pub attributes_to_crop: Option>, - pub crop_length: Option, - pub attributes_to_highlight: Option>, - pub filters: Option, - pub matches: Option, - pub facet_filters: Option, - pub facet_distributions: Option>, -} - -impl SearchQuery { - pub fn perform(&self, index: impl AsRef) -> anyhow::Result { - let index = index.as_ref(); - let before_search = Instant::now(); - let rtxn = index.read_txn()?; - - let mut search = index.search(&rtxn); - - if let Some(ref query) = self.q { - search.query(query); - } - - search.limit(self.limit); - search.offset(self.offset.unwrap_or_default()); - - if let Some(ref facets) = self.facet_filters { - if let Some(facets) = parse_facets(facets, index, &rtxn)? { - search.facet_condition(facets); - } - } - - let milli::SearchResult { - documents_ids, - found_words, - candidates, - } = search.execute()?; - - let mut documents = Vec::new(); - let fields_ids_map = index.fields_ids_map(&rtxn)?; - - let displayed_fields_ids = index.displayed_fields_ids(&rtxn)?; - - let attributes_to_retrieve_ids = match self.attributes_to_retrieve { - Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, - Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), - None => None, - }; - - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { - (_, Some(ids)) => ids, - (Some(ids), None) => ids, - (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let stop_words = fst::Set::default(); - let highlighter = Highlighter::new(&stop_words); - - for (_id, obkv) in index.documents(&rtxn, documents_ids)? { - let mut object = obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv)?; - if let Some(ref attributes_to_highlight) = self.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); - } - documents.push(object); - } - - let nb_hits = candidates.len(); - - let facet_distributions = match self.facet_distributions { - Some(ref fields) => { - let mut facet_distribution = index.facets_distribution(&rtxn); - if fields.iter().all(|f| f != "*") { - facet_distribution.facets(fields); - } - Some(facet_distribution.candidates(candidates).execute()?) - } - None => None, - }; - - Ok(SearchResult { - hits: documents, - nb_hits, - query: self.q.clone().unwrap_or_default(), - limit: self.limit, - offset: self.offset.unwrap_or_default(), - processing_time_ms: before_search.elapsed().as_millis(), - facet_distributions, - }) - } -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SearchResult { - pub hits: Vec>, - pub nb_hits: u64, - pub query: String, - pub limit: usize, - pub offset: usize, - pub processing_time_ms: u128, - #[serde(skip_serializing_if = "Option::is_none")] - pub facet_distributions: Option>>, -} - -pub struct Highlighter<'a, A> { - analyzer: Analyzer<'a, A>, -} - -impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { - pub fn new(stop_words: &'a fst::Set) -> Self { - let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); - Self { analyzer } - } - - pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { - match value { - Value::Null => Value::Null, - Value::Bool(boolean) => Value::Bool(boolean), - Value::Number(number) => Value::Number(number), - Value::String(old_string) => { - let mut string = String::new(); - let analyzed = self.analyzer.analyze(&old_string); - for (word, token) in analyzed.reconstruct() { - if token.is_word() { - let to_highlight = words_to_highlight.contains(token.text()); - if to_highlight { - string.push_str("") - } - string.push_str(word); - if to_highlight { - string.push_str("") - } - } else { - string.push_str(word); - } - } - Value::String(string) - } - Value::Array(values) => Value::Array( - values - .into_iter() - .map(|v| self.highlight_value(v, words_to_highlight)) - .collect(), - ), - Value::Object(object) => Value::Object( - object - .into_iter() - .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) - .collect(), - ), - } - } - - pub fn highlight_record( - &self, - object: &mut Map, - words_to_highlight: &HashSet, - attributes_to_highlight: &HashSet, - ) { - // TODO do we need to create a string for element that are not and needs to be highlight? - for (key, value) in object.iter_mut() { - if attributes_to_highlight.contains(key) { - let old_value = mem::take(value); - *value = self.highlight_value(old_value, words_to_highlight); - } - } - } -} - impl Data { pub async fn search>( &self, @@ -303,49 +108,3 @@ impl Data { //document } } - -fn parse_facets_array( - txn: &RoTxn, - index: &Index, - arr: &Vec, -) -> anyhow::Result> { - let mut ands = Vec::new(); - for value in arr { - match value { - Value::String(s) => ands.push(Either::Right(s.clone())), - Value::Array(arr) => { - let mut ors = Vec::new(); - for value in arr { - match value { - Value::String(s) => ors.push(s.clone()), - v => bail!("Invalid facet expression, expected String, found: {:?}", v), - } - } - ands.push(Either::Left(ors)); - } - v => bail!( - "Invalid facet expression, expected String or [String], found: {:?}", - v - ), - } - } - - FacetCondition::from_array(txn, index, ands) -} - -fn parse_facets( - facets: &Value, - index: &Index, - txn: &RoTxn, -) -> anyhow::Result> { - match facets { - // Disabled for now - //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), - Value::Array(arr) => parse_facets_array(txn, index, arr), - v => bail!( - "Invalid facet expression, expected Array, found: {:?}", - v - ), - } -} - diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index b45b683bf..27d206c06 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -2,25 +2,20 @@ use std::collections::{HashMap, hash_map::Entry}; use std::fs::{File, create_dir_all}; use std::path::{PathBuf, Path}; use std::sync::Arc; -use std::time::Instant; -use anyhow::bail; -use either::Either; use async_stream::stream; use chrono::Utc; use futures::stream::StreamExt; -use heed::{EnvOpenOptions, RoTxn}; +use heed::EnvOpenOptions; use log::info; -use milli::{FacetCondition, Index}; -use serde_json::Value; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use super::update_handler::UpdateHandler; -use crate::data::{SearchQuery, SearchResult}; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; use crate::option::IndexerOpts; +use crate::index::{Index, SearchQuery, SearchResult}; pub type Result = std::result::Result; type AsyncMap = Arc>>; @@ -49,8 +44,8 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; - async fn get_or_create(&self, uuid: Uuid) -> Result>; - async fn get(&self, uuid: Uuid) -> Result>>; + async fn get_or_create(&self, uuid: Uuid) -> Result; + async fn get(&self, uuid: Uuid) -> Result>; } impl IndexActor { @@ -88,7 +83,7 @@ impl IndexActor { async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - let result = perform_search(&index, query); + let result = index.perform_search(query); ret.send(result) }); @@ -104,138 +99,12 @@ impl IndexActor { let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); - let result = tokio::task::spawn_blocking(move || update_handler.handle_update(meta, data, index.as_ref())).await; + let result = tokio::task::spawn_blocking(move || update_handler.handle_update(meta, data, index)).await; let result = result.unwrap(); let _ = ret.send(result); } } -fn perform_search(index: &Index, query: SearchQuery) -> anyhow::Result { - let before_search = Instant::now(); - let rtxn = index.read_txn()?; - - let mut search = index.search(&rtxn); - - if let Some(ref query) = query.q { - search.query(query); - } - - search.limit(query.limit); - search.offset(query.offset.unwrap_or_default()); - - if let Some(ref facets) = query.facet_filters { - if let Some(facets) = parse_facets(facets, index, &rtxn)? { - search.facet_condition(facets); - } - } - - let milli::SearchResult { - documents_ids, - found_words, - candidates, - .. - } = search.execute()?; - let mut documents = Vec::new(); - let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); - - let displayed_fields_ids = index.displayed_fields_ids(&rtxn).unwrap(); - - let attributes_to_retrieve_ids = match query.attributes_to_retrieve { - Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, - Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), - None => None, - }; - - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { - (_, Some(ids)) => ids, - (Some(ids), None) => ids, - (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let stop_words = fst::Set::default(); - let highlighter = crate::data::search::Highlighter::new(&stop_words); - - for (_id, obkv) in index.documents(&rtxn, documents_ids)? { - let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); - if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); - } - documents.push(object); - } - - let nb_hits = candidates.len(); - - let facet_distributions = match query.facet_distributions { - Some(ref fields) => { - let mut facet_distribution = index.facets_distribution(&rtxn); - if fields.iter().all(|f| f != "*") { - facet_distribution.facets(fields); - } - Some(facet_distribution.candidates(candidates).execute()?) - } - None => None, - }; - - let result = SearchResult { - hits: documents, - nb_hits, - query: query.q.clone().unwrap_or_default(), - limit: query.limit, - offset: query.offset.unwrap_or_default(), - processing_time_ms: before_search.elapsed().as_millis(), - facet_distributions, - }; - Ok(result) -} - -fn parse_facets_array( - txn: &RoTxn, - index: &Index, - arr: &Vec, -) -> anyhow::Result> { - let mut ands = Vec::new(); - for value in arr { - match value { - Value::String(s) => ands.push(Either::Right(s.clone())), - Value::Array(arr) => { - let mut ors = Vec::new(); - for value in arr { - match value { - Value::String(s) => ors.push(s.clone()), - v => bail!("Invalid facet expression, expected String, found: {:?}", v), - } - } - ands.push(Either::Left(ors)); - } - v => bail!( - "Invalid facet expression, expected String or [String], found: {:?}", - v - ), - } - } - - FacetCondition::from_array(txn, index, ands) -} - -fn parse_facets( - facets: &Value, - index: &Index, - txn: &RoTxn, -) -> anyhow::Result> { - match facets { - // Disabled for now - //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), - Value::Array(arr) => parse_facets_array(txn, index, arr), - v => bail!( - "Invalid facet expression, expected Array, found: {:?}", - v - ), - } -} #[derive(Clone)] pub struct IndexActorHandle { sender: mpsc::Sender, @@ -276,7 +145,7 @@ impl IndexActorHandle { struct MapIndexStore { root: PathBuf, meta_store: AsyncMap, - index_store: AsyncMap>, + index_store: AsyncMap, } #[async_trait::async_trait] @@ -301,17 +170,18 @@ impl IndexStore for MapIndexStore { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let index = Index::new(options, &db_path) + let index = milli::Index::new(options, &db_path) .map_err(|e| IndexError::Error(e))?; + let index = Index(Arc::new(index)); Ok(index) }).await.expect("thread died"); - self.index_store.write().await.insert(meta.uuid.clone(), Arc::new(index?)); + self.index_store.write().await.insert(meta.uuid.clone(), index?); Ok(meta) } - async fn get_or_create(&self, uuid: Uuid) -> Result> { + async fn get_or_create(&self, uuid: Uuid) -> Result { match self.index_store.write().await.entry(uuid.clone()) { Entry::Vacant(entry) => { match self.meta_store.write().await.entry(uuid.clone()) { @@ -327,7 +197,7 @@ impl IndexStore for MapIndexStore { } } - async fn get(&self, uuid: Uuid) -> Result>> { + async fn get(&self, uuid: Uuid) -> Result> { Ok(self.index_store.read().await.get(&uuid).cloned()) } } diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 882b20963..b893b62e4 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -12,7 +12,7 @@ use super::IndexMetadata; use futures::stream::StreamExt; use actix_web::web::Payload; use super::UpdateMeta; -use crate::data::{SearchResult, SearchQuery}; +use crate::index::{SearchResult, SearchQuery}; use actix_web::web::Bytes; pub struct IndexController { diff --git a/src/index_controller/actor_index_controller/update_handler.rs b/src/index_controller/actor_index_controller/update_handler.rs index d9ac2f866..c42a532ea 100644 --- a/src/index_controller/actor_index_controller/update_handler.rs +++ b/src/index_controller/actor_index_controller/update_handler.rs @@ -7,7 +7,7 @@ use flate2::read::GzDecoder; use grenad::CompressionType; use log::info; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use milli::Index; +use crate::index::Index; use rayon::ThreadPool; use crate::index_controller::updates::{Failed, Processed, Processing}; @@ -225,7 +225,7 @@ impl UpdateHandler { &self, meta: Processing, content: File, - index: &Index, + index: Index, ) -> Result, Failed> { use UpdateMeta::*; @@ -244,12 +244,12 @@ impl UpdateHandler { content, update_builder, primary_key.as_deref(), - index, + &index, ), - ClearDocuments => self.clear_documents(update_builder, index), - DeleteDocuments => self.delete_documents(content, update_builder, index), - Settings(settings) => self.update_settings(settings, update_builder, index), - Facets(levels) => self.update_facets(levels, update_builder, index), + ClearDocuments => self.clear_documents(update_builder, &index), + DeleteDocuments => self.delete_documents(content, update_builder, &index), + Settings(settings) => self.update_settings(settings, update_builder, &index), + Facets(levels) => self.update_facets(levels, update_builder, &index), }; match result { diff --git a/src/lib.rs b/src/lib.rs index 65c4e8eb7..32c8e4266 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ pub mod error; pub mod helpers; pub mod option; pub mod routes; +mod index; mod index_controller; pub use option::Opt; diff --git a/src/routes/search.rs b/src/routes/search.rs index 1f4218555..eb03cf94d 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -4,11 +4,11 @@ use std::convert::{TryFrom, TryInto}; use actix_web::{get, post, web, HttpResponse}; use serde::Deserialize; -use crate::data::{SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; use crate::Data; +use crate::index::{SearchQuery, DEFAULT_SEARCH_LIMIT}; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(search_with_post).service(search_with_url_query); From a56db854a215eaa7f7c39a7984602d73f1c6385e Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 11:56:32 +0100 Subject: [PATCH 114/522] refactor update handler --- src/data/mod.rs | 3 +- src/data/updates.rs | 5 +- src/index/mod.rs | 19 ++ src/index/search.rs | 245 ++++++++++++++++++ src/index/updates.rs | 229 ++++++++++++++++ .../actor_index_controller/index_actor.rs | 3 +- .../actor_index_controller/mod.rs | 9 +- .../actor_index_controller/update_actor.rs | 3 +- .../actor_index_controller/update_handler.rs | 178 +------------ src/index_controller/mod.rs | 68 +---- src/routes/settings/mod.rs | 6 +- 11 files changed, 522 insertions(+), 246 deletions(-) create mode 100644 src/index/mod.rs create mode 100644 src/index/search.rs create mode 100644 src/index/updates.rs diff --git a/src/data/mod.rs b/src/data/mod.rs index 58acb105a..79572fcf7 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -7,8 +7,9 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexMetadata, Settings, IndexSettings}; +use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::index_controller::actor_index_controller::IndexController; +use crate::index::Settings; use crate::option::Opt; #[derive(Clone)] diff --git a/src/data/updates.rs b/src/data/updates.rs index 01f5174a2..9f3e9b3fd 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -4,10 +4,11 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; //use tokio::io::AsyncWriteExt; use actix_web::web::Payload; -use crate::index_controller::UpdateStatus; -use crate::index_controller::{Settings, IndexMetadata}; +use crate::index_controller::{UpdateStatus, IndexMetadata}; +use crate::index::Settings; use super::Data; + impl Data { pub async fn add_documents( &self, diff --git a/src/index/mod.rs b/src/index/mod.rs new file mode 100644 index 000000000..f35e6b3dd --- /dev/null +++ b/src/index/mod.rs @@ -0,0 +1,19 @@ +mod search; +mod updates; + +use std::sync::Arc; +use std::ops::Deref; + +pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; +pub use updates::{Settings, Facets, UpdateResult}; + +#[derive(Clone)] +pub struct Index(pub Arc); + +impl Deref for Index { + type Target = milli::Index; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} diff --git a/src/index/search.rs b/src/index/search.rs new file mode 100644 index 000000000..1264d49d6 --- /dev/null +++ b/src/index/search.rs @@ -0,0 +1,245 @@ +use std::time::Instant; +use std::collections::{HashSet, BTreeMap}; +use std::mem; + +use either::Either; +use anyhow::bail; +use heed::RoTxn; +use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use milli::{FacetCondition, facet::FacetValue}; +use serde::{Serialize, Deserialize}; +use serde_json::{Value, Map}; + +use super::Index; + +pub const DEFAULT_SEARCH_LIMIT: usize = 20; + +const fn default_search_limit() -> usize { + DEFAULT_SEARCH_LIMIT +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[allow(dead_code)] +pub struct SearchQuery { + pub q: Option, + pub offset: Option, + #[serde(default = "default_search_limit")] + pub limit: usize, + pub attributes_to_retrieve: Option>, + pub attributes_to_crop: Option>, + pub crop_length: Option, + pub attributes_to_highlight: Option>, + pub filters: Option, + pub matches: Option, + pub facet_filters: Option, + pub facet_distributions: Option>, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchResult { + pub hits: Vec>, + pub nb_hits: u64, + pub query: String, + pub limit: usize, + pub offset: usize, + pub processing_time_ms: u128, + #[serde(skip_serializing_if = "Option::is_none")] + pub facet_distributions: Option>>, +} + +impl Index { + pub fn perform_search(&self, query: SearchQuery) -> anyhow::Result { + let before_search = Instant::now(); + let rtxn = self.read_txn()?; + + let mut search = self.search(&rtxn); + + if let Some(ref query) = query.q { + search.query(query); + } + + search.limit(query.limit); + search.offset(query.offset.unwrap_or_default()); + + if let Some(ref facets) = query.facet_filters { + if let Some(facets) = parse_facets(facets, self, &rtxn)? { + search.facet_condition(facets); + } + } + + let milli::SearchResult { + documents_ids, + found_words, + candidates, + .. + } = search.execute()?; + let mut documents = Vec::new(); + let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields_ids = self.displayed_fields_ids(&rtxn).unwrap(); + + let attributes_to_retrieve_ids = match query.attributes_to_retrieve { + Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, + Some(ref attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>() + .into(), + None => None, + }; + + let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { + (_, Some(ids)) => ids, + (Some(ids), None) => ids, + (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new(&stop_words); + + for (_id, obkv) in self.documents(&rtxn, documents_ids)? { + let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); + if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { + highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + } + documents.push(object); + } + + let nb_hits = candidates.len(); + + let facet_distributions = match query.facet_distributions { + Some(ref fields) => { + let mut facet_distribution = self.facets_distribution(&rtxn); + if fields.iter().all(|f| f != "*") { + facet_distribution.facets(fields); + } + Some(facet_distribution.candidates(candidates).execute()?) + } + None => None, + }; + + let result = SearchResult { + hits: documents, + nb_hits, + query: query.q.clone().unwrap_or_default(), + limit: query.limit, + offset: query.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + facet_distributions, + }; + Ok(result) + } +} + +fn parse_facets_array( + txn: &RoTxn, + index: &Index, + arr: &Vec, +) -> anyhow::Result> { + let mut ands = Vec::new(); + for value in arr { + match value { + Value::String(s) => ands.push(Either::Right(s.clone())), + Value::Array(arr) => { + let mut ors = Vec::new(); + for value in arr { + match value { + Value::String(s) => ors.push(s.clone()), + v => bail!("Invalid facet expression, expected String, found: {:?}", v), + } + } + ands.push(Either::Left(ors)); + } + v => bail!( + "Invalid facet expression, expected String or [String], found: {:?}", + v + ), + } + } + + FacetCondition::from_array(txn, &index.0, ands) +} + +pub struct Highlighter<'a, A> { + analyzer: Analyzer<'a, A>, +} + +impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { + pub fn new(stop_words: &'a fst::Set) -> Self { + let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); + + Self { analyzer } + } + + pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + match value { + Value::Null => Value::Null, + Value::Bool(boolean) => Value::Bool(boolean), + Value::Number(number) => Value::Number(number), + Value::String(old_string) => { + let mut string = String::new(); + let analyzed = self.analyzer.analyze(&old_string); + for (word, token) in analyzed.reconstruct() { + if token.is_word() { + let to_highlight = words_to_highlight.contains(token.text()); + if to_highlight { + string.push_str("") + } + string.push_str(word); + if to_highlight { + string.push_str("") + } + } else { + string.push_str(word); + } + } + Value::String(string) + } + Value::Array(values) => Value::Array( + values + .into_iter() + .map(|v| self.highlight_value(v, words_to_highlight)) + .collect(), + ), + Value::Object(object) => Value::Object( + object + .into_iter() + .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) + .collect(), + ), + } + } + + pub fn highlight_record( + &self, + object: &mut Map, + words_to_highlight: &HashSet, + attributes_to_highlight: &HashSet, + ) { + // TODO do we need to create a string for element that are not and needs to be highlight? + for (key, value) in object.iter_mut() { + if attributes_to_highlight.contains(key) { + let old_value = mem::take(value); + *value = self.highlight_value(old_value, words_to_highlight); + } + } + } +} + +fn parse_facets( + facets: &Value, + index: &Index, + txn: &RoTxn, +) -> anyhow::Result> { + match facets { + // Disabled for now + //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), + Value::Array(arr) => parse_facets_array(txn, index, arr), + v => bail!( + "Invalid facet expression, expected Array, found: {:?}", + v + ), + } +} diff --git a/src/index/updates.rs b/src/index/updates.rs new file mode 100644 index 000000000..d339406f7 --- /dev/null +++ b/src/index/updates.rs @@ -0,0 +1,229 @@ +use std::collections::HashMap; +use std::io; +use std::num::NonZeroUsize; + +use flate2::read::GzDecoder; +use log::info; +use milli::update::{UpdateFormat, IndexDocumentsMethod, UpdateBuilder, DocumentAdditionResult}; +use serde::{Serialize, Deserialize, de::Deserializer}; + +use super::Index; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum UpdateResult { + DocumentsAddition(DocumentAdditionResult), + DocumentDeletion { deleted: usize }, + Other, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Settings { + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub displayed_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub searchable_attributes: Option>>, + + #[serde(default)] + pub faceted_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub criteria: Option>>, +} + +impl Settings { + pub fn cleared() -> Self { + Self { + displayed_attributes: Some(None), + searchable_attributes: Some(None), + faceted_attributes: Some(None), + criteria: Some(None), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Facets { + pub level_group_size: Option, + pub min_level_size: Option, +} + +fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> +where T: Deserialize<'de>, + D: Deserializer<'de> +{ + Deserialize::deserialize(deserializer).map(Some) +} + +impl Index { + pub fn update_documents( + &self, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: impl io::Read, + update_builder: UpdateBuilder, + primary_key: Option<&str>, + ) -> anyhow::Result { + info!("performing document addition"); + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + + // Set the primary key if not set already, ignore if already set. + match (self.primary_key(&wtxn)?, primary_key) { + (None, Some(ref primary_key)) => { + self.put_primary_key(&mut wtxn, primary_key)?; + } + _ => (), + } + + let mut builder = update_builder.index_documents(&mut wtxn, self); + builder.update_format(format); + builder.index_documents_method(method); + + let gzipped = false; + let reader = if gzipped { + Box::new(GzDecoder::new(content)) + } else { + Box::new(content) as Box + }; + + let result = builder.execute(reader, |indexing_step, update_id| { + info!("update {}: {:?}", update_id, indexing_step) + }); + + info!("document addition done: {:?}", result); + + match result { + Ok(addition_result) => wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + pub fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, self); + + match builder.execute() { + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + pub fn update_settings( + &self, + settings: &Settings, + update_builder: UpdateBuilder, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + let mut builder = update_builder.settings(&mut wtxn, self); + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names.clone()), + None => builder.reset_searchable_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names.clone()), + None => builder.reset_displayed_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref facet_types) = settings.faceted_attributes { + let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + builder.set_faceted_fields(facet_types); + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref criteria) = settings.criteria { + match criteria { + Some(criteria) => builder.set_criteria(criteria.clone()), + None => builder.reset_criteria(), + } + } + + let result = builder + .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + pub fn update_facets( + &self, + levels: &Facets, + update_builder: UpdateBuilder, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + let mut builder = update_builder.facets(&mut wtxn, self); + if let Some(value) = levels.level_group_size { + builder.level_group_size(value); + } + if let Some(value) = levels.min_level_size { + builder.min_level_size(value); + } + match builder.execute() { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + pub fn delete_documents( + &self, + document_ids: impl io::Read, + update_builder: UpdateBuilder, + ) -> anyhow::Result { + let ids: Vec = serde_json::from_reader(document_ids)?; + let mut txn = self.write_txn()?; + let mut builder = update_builder.delete_documents(&mut txn, self)?; + + // We ignore unexisting document ids + ids.iter().for_each(|id| { builder.delete_external_id(id); }); + + match builder.execute() { + Ok(deleted) => txn + .commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } +} diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 27d206c06..96e5010db 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -13,7 +13,8 @@ use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use super::update_handler::UpdateHandler; -use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; +use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}}; +use crate::index::UpdateResult as UResult; use crate::option::IndexerOpts; use crate::index::{Index, SearchQuery, SearchResult}; diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index b893b62e4..188d85580 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -15,6 +15,9 @@ use super::UpdateMeta; use crate::index::{SearchResult, SearchQuery}; use actix_web::web::Bytes; +use crate::index::Settings; +use super::UpdateStatus; + pub struct IndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, @@ -69,7 +72,7 @@ impl IndexController { Ok(status) } - fn clear_documents(&self, index: String) -> anyhow::Result { + fn clear_documents(&self, index: String) -> anyhow::Result { todo!() } @@ -77,7 +80,7 @@ impl IndexController { todo!() } - fn update_settings(&self, index_uid: String, settings: super::Settings) -> anyhow::Result { + fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { todo!() } @@ -100,7 +103,7 @@ impl IndexController { todo!() } - fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + fn update_status(&self, index: String, id: u64) -> anyhow::Result> { todo!() } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index e82f01092..384c52098 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -10,7 +10,8 @@ use uuid::Uuid; use tokio::fs::File; use tokio::io::AsyncWriteExt; -use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult}; +use crate::index_controller::{UpdateMeta, UpdateStatus}; +use crate::index::UpdateResult; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; diff --git a/src/index_controller/actor_index_controller/update_handler.rs b/src/index_controller/actor_index_controller/update_handler.rs index c42a532ea..766dfc5f0 100644 --- a/src/index_controller/actor_index_controller/update_handler.rs +++ b/src/index_controller/actor_index_controller/update_handler.rs @@ -1,17 +1,14 @@ -use std::collections::HashMap; -use std::io; use std::fs::File; use anyhow::Result; -use flate2::read::GzDecoder; use grenad::CompressionType; -use log::info; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::update::UpdateBuilder; use crate::index::Index; use rayon::ThreadPool; use crate::index_controller::updates::{Failed, Processed, Processing}; -use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; +use crate::index_controller::UpdateMeta; +use crate::index::UpdateResult; use crate::option::IndexerOpts; pub struct UpdateHandler { @@ -62,164 +59,6 @@ impl UpdateHandler { update_builder } - fn update_documents( - &self, - format: UpdateFormat, - method: IndexDocumentsMethod, - content: File, - update_builder: UpdateBuilder, - primary_key: Option<&str>, - index: &Index, - ) -> anyhow::Result { - info!("performing document addition"); - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - - // Set the primary key if not set already, ignore if already set. - match (index.primary_key(&wtxn)?, primary_key) { - (None, Some(ref primary_key)) => { - index.put_primary_key(&mut wtxn, primary_key)?; - } - _ => (), - } - - let mut builder = update_builder.index_documents(&mut wtxn, index); - builder.update_format(format); - builder.index_documents_method(method); - - let gzipped = false; - let reader = if gzipped { - Box::new(GzDecoder::new(content)) - } else { - Box::new(content) as Box - }; - - let result = builder.execute(reader, |indexing_step, update_id| { - info!("update {}: {:?}", update_id, indexing_step) - }); - - info!("document addition done: {:?}", result); - - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn clear_documents(&self, update_builder: UpdateBuilder, index: &Index) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let builder = update_builder.clear_documents(&mut wtxn, index); - - match builder.execute() { - Ok(_count) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_settings( - &self, - settings: &Settings, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let mut builder = update_builder.settings(&mut wtxn, index); - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.searchable_attributes { - match names { - Some(names) => builder.set_searchable_fields(names.clone()), - None => builder.reset_searchable_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.displayed_attributes { - match names { - Some(names) => builder.set_displayed_fields(names.clone()), - None => builder.reset_displayed_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); - builder.set_faceted_fields(facet_types); - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { - match criteria { - Some(criteria) => builder.set_criteria(criteria.clone()), - None => builder.reset_criteria(), - } - } - - let result = builder - .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_facets( - &self, - levels: &Facets, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let mut builder = update_builder.facets(&mut wtxn, index); - if let Some(value) = levels.level_group_size { - builder.level_group_size(value); - } - if let Some(value) = levels.min_level_size { - builder.min_level_size(value); - } - match builder.execute() { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn delete_documents( - &self, - document_ids: File, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - let ids: Vec = serde_json::from_reader(document_ids)?; - let mut txn = index.write_txn()?; - let mut builder = update_builder.delete_documents(&mut txn, index)?; - - // We ignore unexisting document ids - ids.iter().for_each(|id| { builder.delete_external_id(id); }); - - match builder.execute() { - Ok(deleted) => txn - .commit() - .and(Ok(UpdateResult::DocumentDeletion { deleted })) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } pub fn handle_update( &self, @@ -238,18 +77,17 @@ impl UpdateHandler { method, format, primary_key, - } => self.update_documents( + } => index.update_documents( *format, *method, content, update_builder, primary_key.as_deref(), - &index, ), - ClearDocuments => self.clear_documents(update_builder, &index), - DeleteDocuments => self.delete_documents(content, update_builder, &index), - Settings(settings) => self.update_settings(settings, update_builder, &index), - Facets(levels) => self.update_facets(levels, update_builder, &index), + ClearDocuments => index.clear_documents(update_builder), + DeleteDocuments => index.delete_documents(content, update_builder), + Settings(settings) => index.update_settings(settings, update_builder), + Facets(levels) => index.update_facets(levels, update_builder), }; match result { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index fbe7a161d..16f884137 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,16 +1,13 @@ pub mod actor_index_controller; mod updates; -use std::collections::HashMap; -use std::num::NonZeroUsize; - -use anyhow::Result; use chrono::{DateTime, Utc}; -use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; -use serde::{Serialize, Deserialize, de::Deserializer}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use serde::{Serialize, Deserialize}; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; +use crate::index::{UpdateResult, Settings, Facets}; pub type UpdateStatus = updates::UpdateStatus; @@ -37,66 +34,7 @@ pub enum UpdateMeta { Facets(Facets), } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct Facets { - pub level_group_size: Option, - pub min_level_size: Option, -} -fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> -where T: Deserialize<'de>, - D: Deserializer<'de> -{ - Deserialize::deserialize(deserializer).map(Some) -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct Settings { - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub displayed_attributes: Option>>, - - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub searchable_attributes: Option>>, - - #[serde(default)] - pub faceted_attributes: Option>>, - - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub criteria: Option>>, -} - -impl Settings { - pub fn cleared() -> Self { - Self { - displayed_attributes: Some(None), - searchable_attributes: Some(None), - faceted_attributes: Some(None), - criteria: Some(None), - } - } -} -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateResult { - DocumentsAddition(DocumentAdditionResult), - DocumentDeletion { deleted: usize }, - Other, -} #[derive(Clone, Debug)] pub struct IndexSettings { diff --git a/src/routes/settings/mod.rs b/src/routes/settings/mod.rs index 00bc4220e..93e10674d 100644 --- a/src/routes/settings/mod.rs +++ b/src/routes/settings/mod.rs @@ -2,7 +2,7 @@ use actix_web::{web, HttpResponse, delete, get, post}; use crate::Data; use crate::error::ResponseError; -use crate::index_controller::Settings; +use crate::index::Settings; use crate::helpers::Authentication; #[macro_export] @@ -14,14 +14,14 @@ macro_rules! make_setting_route { use crate::data; use crate::error::ResponseError; use crate::helpers::Authentication; - use crate::index_controller::Settings; + use crate::index::Settings; #[actix_web::delete($route, wrap = "Authentication::Private")] pub async fn delete( data: web::Data, index_uid: web::Path, ) -> Result { - use crate::index_controller::Settings; + use crate::index::Settings; let settings = Settings { $attr: Some(None), ..Default::default() From 8432c8584a6fee1a555454693f0e9e5741d61d69 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 12:03:06 +0100 Subject: [PATCH 115/522] refactor index controller --- src/data/mod.rs | 2 +- .../index_actor.rs | 0 src/index_controller/mod.rs | 124 +++++++++++++++++- .../update_actor.rs | 0 .../update_handler.rs | 0 .../update_store.rs | 0 .../uuid_resolver.rs | 0 7 files changed, 122 insertions(+), 4 deletions(-) rename src/index_controller/{actor_index_controller => }/index_actor.rs (100%) rename src/index_controller/{actor_index_controller => }/update_actor.rs (100%) rename src/index_controller/{actor_index_controller => }/update_handler.rs (100%) rename src/index_controller/{actor_index_controller => }/update_store.rs (100%) rename src/index_controller/{actor_index_controller => }/uuid_resolver.rs (100%) diff --git a/src/data/mod.rs b/src/data/mod.rs index 79572fcf7..76effde58 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use sha2::Digest; use crate::index_controller::{IndexMetadata, IndexSettings}; -use crate::index_controller::actor_index_controller::IndexController; +use crate::index_controller::IndexController; use crate::index::Settings; use crate::option::Opt; diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/index_actor.rs similarity index 100% rename from src/index_controller/actor_index_controller/index_actor.rs rename to src/index_controller/index_actor.rs diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 16f884137..54199c1d7 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,6 +1,17 @@ -pub mod actor_index_controller; mod updates; +mod index_actor; +mod update_actor; +mod uuid_resolver; +mod update_store; +mod update_handler; +use std::path::Path; + +use tokio::sync::{mpsc, oneshot}; +use futures::stream::StreamExt; +use actix_web::web::Payload; +use crate::index::{SearchResult, SearchQuery}; +use actix_web::web::Bytes; use chrono::{DateTime, Utc}; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; @@ -34,10 +45,117 @@ pub enum UpdateMeta { Facets(Facets), } - - #[derive(Clone, Debug)] pub struct IndexSettings { pub name: Option, pub primary_key: Option, } + + +pub struct IndexController { + uuid_resolver: uuid_resolver::UuidResolverHandle, + index_handle: index_actor::IndexActorHandle, + update_handle: update_actor::UpdateActorHandle, +} + +enum IndexControllerMsg { + CreateIndex { + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + }, + Shutdown, +} + +impl IndexController { + pub fn new(path: impl AsRef) -> Self { + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); + let index_actor = index_actor::IndexActorHandle::new(&path); + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); + Self { uuid_resolver, index_handle: index_actor, update_handle } + } + + pub async fn add_documents( + &self, + index: String, + method: milli::update::IndexDocumentsMethod, + format: milli::update::UpdateFormat, + mut payload: Payload, + primary_key: Option, + ) -> anyhow::Result { + let uuid = self.uuid_resolver.get_or_create(index).await?; + let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; + let (sender, receiver) = mpsc::channel(10); + + // It is necessary to spawn a local task to senf the payload to the update handle to + // prevent dead_locking between the update_handle::update that waits for the update to be + // registered and the update_actor that waits for the the payload to be sent to it. + tokio::task::spawn_local(async move { + while let Some(bytes) = payload.next().await { + match bytes { + Ok(bytes) => { sender.send(Ok(bytes)).await; }, + Err(e) => { + let error: Box = Box::new(e); + sender.send(Err(error)).await; }, + } + } + }); + + // This must be done *AFTER* spawning the task. + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) + } + + fn clear_documents(&self, index: String) -> anyhow::Result { + todo!() + } + + fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { + todo!() + } + + fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { + todo!() + } + + pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { + let IndexSettings { name, primary_key } = index_settings; + let uuid = self.uuid_resolver.create(name.unwrap()).await?; + let index_meta = self.index_handle.create_index(uuid, primary_key).await?; + Ok(index_meta) + } + + fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { + todo!() + } + + fn swap_indices(&self, index1_uid: String, index2_uid: String) -> anyhow::Result<()> { + todo!() + } + + pub fn index(&self, name: String) -> anyhow::Result>> { + todo!() + } + + fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + todo!() + } + + fn all_update_status(&self, index: String) -> anyhow::Result> { + todo!() + } + + pub fn list_indexes(&self) -> anyhow::Result> { + todo!() + } + + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { + todo!() + } + + pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); + let result = self.index_handle.search(uuid, query).await?; + Ok(result) + } +} diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/update_actor.rs similarity index 100% rename from src/index_controller/actor_index_controller/update_actor.rs rename to src/index_controller/update_actor.rs diff --git a/src/index_controller/actor_index_controller/update_handler.rs b/src/index_controller/update_handler.rs similarity index 100% rename from src/index_controller/actor_index_controller/update_handler.rs rename to src/index_controller/update_handler.rs diff --git a/src/index_controller/actor_index_controller/update_store.rs b/src/index_controller/update_store.rs similarity index 100% rename from src/index_controller/actor_index_controller/update_store.rs rename to src/index_controller/update_store.rs diff --git a/src/index_controller/actor_index_controller/uuid_resolver.rs b/src/index_controller/uuid_resolver.rs similarity index 100% rename from src/index_controller/actor_index_controller/uuid_resolver.rs rename to src/index_controller/uuid_resolver.rs From 47138c763290fe5274985358c365eac7c622cb63 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 12:20:14 +0100 Subject: [PATCH 116/522] update settings --- src/data/updates.rs | 10 ++++------ src/index_controller/mod.rs | 10 ++++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 9f3e9b3fd..92bba693c 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -25,13 +25,11 @@ impl Data { pub async fn update_settings( &self, - _index: impl AsRef + Send + Sync + 'static, - _settings: Settings + index: String, + settings: Settings ) -> anyhow::Result { - todo!() - //let index_controller = self.index_controller.clone(); - //let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; - //Ok(update.into()) + let update = self.index_controller.update_settings(index, settings).await?; + Ok(update.into()) } pub async fn clear_documents( diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 54199c1d7..dd2dbd7cf 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -114,8 +114,14 @@ impl IndexController { todo!() } - fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { - todo!() + pub async fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { + let uuid = self.uuid_resolver.get_or_create(index_uid).await?; + let meta = UpdateMeta::Settings(settings); + // Nothing so send, drop the sender right away, as not to block the update actor. + let (_, receiver) = mpsc::channel(1); + + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) } pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { From 17b84691f2d3c10354915ac889798baab9e28e93 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 12:38:55 +0100 Subject: [PATCH 117/522] list settings --- src/data/mod.rs | 31 ++--------------------------- src/index/mod.rs | 29 +++++++++++++++++++++++++++ src/index_controller/index_actor.rs | 27 +++++++++++++++++++++---- src/index_controller/mod.rs | 26 +++++++++++++----------- src/routes/settings/mod.rs | 4 ++-- 5 files changed, 70 insertions(+), 47 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 76effde58..1dec00766 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -78,35 +78,8 @@ impl Data { Ok(Data { inner }) } - pub fn settings>(&self, index_uid: S) -> anyhow::Result { - let index = self.index_controller - .index(index_uid.as_ref().to_string())? - .ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?; - - let txn = index.read_txn()?; - - let displayed_attributes = index - .displayed_fields(&txn)? - .map(|fields| fields.into_iter().map(String::from).collect()) - .unwrap_or_else(|| vec!["*".to_string()]); - - let searchable_attributes = index - .searchable_fields(&txn)? - .map(|fields| fields.into_iter().map(String::from).collect()) - .unwrap_or_else(|| vec!["*".to_string()]); - - let faceted_attributes = index - .faceted_fields(&txn)? - .into_iter() - .map(|(k, v)| (k, v.to_string())) - .collect(); - - Ok(Settings { - displayed_attributes: Some(Some(displayed_attributes)), - searchable_attributes: Some(Some(searchable_attributes)), - faceted_attributes: Some(Some(faceted_attributes)), - criteria: None, - }) + pub async fn settings>(&self, index_uid: S) -> anyhow::Result { + self.index_controller.settings(index_uid.as_ref().to_string()).await } pub fn list_indexes(&self) -> anyhow::Result> { diff --git a/src/index/mod.rs b/src/index/mod.rs index f35e6b3dd..a68f983e9 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -17,3 +17,32 @@ impl Deref for Index { self.0.as_ref() } } + +impl Index { + pub fn settings(&self) -> anyhow::Result { + let txn = self.read_txn()?; + + let displayed_attributes = self + .displayed_fields(&txn)? + .map(|fields| fields.into_iter().map(String::from).collect()) + .unwrap_or_else(|| vec!["*".to_string()]); + + let searchable_attributes = self + .searchable_fields(&txn)? + .map(|fields| fields.into_iter().map(String::from).collect()) + .unwrap_or_else(|| vec!["*".to_string()]); + + let faceted_attributes = self + .faceted_fields(&txn)? + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(); + + Ok(Settings { + displayed_attributes: Some(Some(displayed_attributes)), + searchable_attributes: Some(Some(searchable_attributes)), + faceted_attributes: Some(Some(faceted_attributes)), + criteria: None, + }) + } +} diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 96e5010db..655a1e2d5 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -16,7 +16,7 @@ use super::update_handler::UpdateHandler; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}}; use crate::index::UpdateResult as UResult; use crate::option::IndexerOpts; -use crate::index::{Index, SearchQuery, SearchResult}; +use crate::index::{Index, SearchQuery, SearchResult, Settings}; pub type Result = std::result::Result; type AsyncMap = Arc>>; @@ -26,6 +26,7 @@ enum IndexMsg { CreateIndex { uuid: Uuid, primary_key: Option, ret: oneshot::Sender> }, Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, Search { uuid: Uuid, query: SearchQuery, ret: oneshot::Sender> }, + Settings { uuid: Uuid, ret: oneshot::Sender> }, } struct IndexActor { @@ -75,6 +76,7 @@ impl IndexActor { IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret).await, IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret).await, IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, + IndexMsg::Settings { ret, uuid } => self.handle_settings(uuid, ret).await, } }); @@ -100,9 +102,19 @@ impl IndexActor { let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); - let result = tokio::task::spawn_blocking(move || update_handler.handle_update(meta, data, index)).await; - let result = result.unwrap(); - let _ = ret.send(result); + tokio::task::spawn_blocking(move || { + let result = update_handler.handle_update(meta, data, index); + let _ = ret.send(result); + }).await; + } + + async fn handle_settings(&self, uuid: Uuid, ret: oneshot::Sender>) { + let index = self.store.get(uuid).await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || { + let result = index.settings() + .map_err(|e| IndexError::Error(e)); + let _ = ret.send(result); + }).await; } } @@ -141,6 +153,13 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn settings(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Settings { uuid, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index dd2dbd7cf..f1bc922d6 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -7,14 +7,15 @@ mod update_handler; use std::path::Path; -use tokio::sync::{mpsc, oneshot}; -use futures::stream::StreamExt; -use actix_web::web::Payload; -use crate::index::{SearchResult, SearchQuery}; use actix_web::web::Bytes; +use actix_web::web::Payload; +use anyhow::Context; use chrono::{DateTime, Utc}; +use crate::index::{SearchResult, SearchQuery}; +use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; +use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; @@ -135,14 +136,6 @@ impl IndexController { todo!() } - fn swap_indices(&self, index1_uid: String, index2_uid: String) -> anyhow::Result<()> { - todo!() - } - - pub fn index(&self, name: String) -> anyhow::Result>> { - todo!() - } - fn update_status(&self, index: String, id: u64) -> anyhow::Result> { todo!() } @@ -155,6 +148,15 @@ impl IndexController { todo!() } + pub async fn settings(&self, index: String) -> anyhow::Result { + let uuid = self.uuid_resolver + .resolve(index.clone()) + .await? + .with_context(|| format!("Index {:?} doesn't exist", index))?; + let settings = self.index_handle.settings(uuid).await?; + Ok(settings) + } + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { todo!() } diff --git a/src/routes/settings/mod.rs b/src/routes/settings/mod.rs index 93e10674d..0f904f572 100644 --- a/src/routes/settings/mod.rs +++ b/src/routes/settings/mod.rs @@ -64,7 +64,7 @@ macro_rules! make_setting_route { data: actix_web::web::Data, index_uid: actix_web::web::Path, ) -> std::result::Result { - match data.settings(index_uid.as_ref()) { + match data.settings(index_uid.as_ref()).await { Ok(settings) => { let setting = settings.$attr; let json = serde_json::to_string(&setting).unwrap(); @@ -153,7 +153,7 @@ async fn get_all( data: web::Data, index_uid: web::Path, ) -> Result { - match data.settings(index_uid.as_ref()) { + match data.settings(index_uid.as_ref()).await { Ok(settings) => { let json = serde_json::to_string(&settings).unwrap(); Ok(HttpResponse::Ok().body(json)) From f3d65ec5e92f6a3307c6eae0a2eb488e2ade9d4c Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 14:20:19 +0100 Subject: [PATCH 118/522] implement retrieve documents --- src/data/search.rs | 53 ++------ src/index/mod.rs | 39 ++++++ src/index_controller/index_actor.rs | 187 ++++++++++++++++++++++------ src/index_controller/mod.rs | 17 ++- src/routes/document.rs | 23 ++-- 5 files changed, 225 insertions(+), 94 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index d6bf8438e..319f8b973 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -12,59 +12,22 @@ impl Data { self.index_controller.search(index.as_ref().to_string(), search_query).await } - pub async fn retrieve_documents( + pub async fn retrieve_documents( &self, - _index: String, - _offset: usize, - _limit: usize, - _attributes_to_retrieve: Option>, - ) -> anyhow::Result>> - where - S: AsRef + Send + Sync + 'static, - { - todo!() - //let index_controller = self.index_controller.clone(); - //let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { - //let index = index_controller - //.index(index.clone())? - //.with_context(|| format!("Index {:?} doesn't exist", index))?; - - //let txn = index.read_txn()?; - - //let fields_ids_map = index.fields_ids_map(&txn)?; - - //let attributes_to_retrieve_ids = match attributes_to_retrieve { - //Some(attrs) => attrs - //.iter() - //.filter_map(|f| fields_ids_map.id(f.as_ref())) - //.collect::>(), - //None => fields_ids_map.iter().map(|(id, _)| id).collect(), - //}; - - //let iter = index.documents.range(&txn, &(..))?.skip(offset).take(limit); - - //let mut documents = Vec::new(); - - //for entry in iter { - //let (_id, obkv) = entry?; - //let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; - //documents.push(object); - //} - - //Ok(documents) - //}) - //.await?; - //documents + index: String, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> anyhow::Result>> { + self.index_controller.documents(index, offset, limit, attributes_to_retrieve).await } pub async fn retrieve_document( &self, _index: impl AsRef + Sync + Send + 'static, _document_id: impl AsRef + Sync + Send + 'static, - _attributes_to_retrieve: Option>, + _attributes_to_retrieve: Option>, ) -> anyhow::Result> - where - S: AsRef + Sync + Send + 'static, { todo!() //let index_controller = self.index_controller.clone(); diff --git a/src/index/mod.rs b/src/index/mod.rs index a68f983e9..8ae2ba6a2 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -4,9 +4,14 @@ mod updates; use std::sync::Arc; use std::ops::Deref; +use serde_json::{Value, Map}; +use milli::obkv_to_json; + pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use updates::{Settings, Facets, UpdateResult}; +pub type Document = Map; + #[derive(Clone)] pub struct Index(pub Arc); @@ -45,4 +50,38 @@ impl Index { criteria: None, }) } + + pub fn retrieve_documents( + &self, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> anyhow::Result>> + where + S: AsRef + Send + Sync + 'static, + { + let txn = self.read_txn()?; + + let fields_ids_map = self.fields_ids_map(&txn)?; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); + + let mut documents = Vec::new(); + + for entry in iter { + let (_id, obkv) = entry?; + let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; + documents.push(object); + } + + Ok(documents) + } } diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 655a1e2d5..8e59227ac 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -1,6 +1,6 @@ -use std::collections::{HashMap, hash_map::Entry}; -use std::fs::{File, create_dir_all}; -use std::path::{PathBuf, Path}; +use std::collections::{hash_map::Entry, HashMap}; +use std::fs::{create_dir_all, File}; +use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; @@ -8,25 +8,51 @@ use chrono::Utc; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; +use serde_json::{Map, Value}; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use super::update_handler::UpdateHandler; -use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}}; use crate::index::UpdateResult as UResult; +use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; +use crate::index_controller::{ + updates::{Failed, Processed, Processing}, + IndexMetadata, UpdateMeta, +}; use crate::option::IndexerOpts; -use crate::index::{Index, SearchQuery, SearchResult, Settings}; pub type Result = std::result::Result; type AsyncMap = Arc>>; type UpdateResult = std::result::Result, Failed>; enum IndexMsg { - CreateIndex { uuid: Uuid, primary_key: Option, ret: oneshot::Sender> }, - Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, - Search { uuid: Uuid, query: SearchQuery, ret: oneshot::Sender> }, - Settings { uuid: Uuid, ret: oneshot::Sender> }, + CreateIndex { + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + }, + Update { + meta: Processing, + data: std::fs::File, + ret: oneshot::Sender, + }, + Search { + uuid: Uuid, + query: SearchQuery, + ret: oneshot::Sender>, + }, + Settings { + uuid: Uuid, + ret: oneshot::Sender>, + }, + Documents { + uuid: Uuid, + attributes_to_retrieve: Option>, + offset: usize, + limit: usize, + ret: oneshot::Sender>>>, + }, } struct IndexActor { @@ -56,11 +82,20 @@ impl IndexActor { let update_handler = UpdateHandler::new(&options).unwrap(); let update_handler = Arc::new(update_handler); let inbox = Some(inbox); - Self { inbox, store, update_handler } + Self { + inbox, + store, + update_handler, + } } async fn run(mut self) { - let mut inbox = self.inbox.take().expect("Index Actor must have a inbox at this point."); + use IndexMsg::*; + + let mut inbox = self + .inbox + .take() + .expect("Index Actor must have a inbox at this point."); let stream = stream! { loop { @@ -73,31 +108,59 @@ impl IndexActor { let fut = stream.for_each_concurrent(Some(10), |msg| async { match msg { - IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret).await, - IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret).await, - IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, - IndexMsg::Settings { ret, uuid } => self.handle_settings(uuid, ret).await, + CreateIndex { + uuid, + primary_key, + ret, + } => self.handle_create_index(uuid, primary_key, ret).await, + Update { ret, meta, data } => self.handle_update(meta, data, ret).await, + Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, + Settings { ret, uuid } => self.handle_settings(uuid, ret).await, + Documents { + ret, + uuid, + attributes_to_retrieve, + offset, + limit, + } => { + self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) + .await + } } }); fut.await; } - async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { + async fn handle_search( + &self, + uuid: Uuid, + query: SearchQuery, + ret: oneshot::Sender>, + ) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { let result = index.perform_search(query); ret.send(result) }); - } - async fn handle_create_index(&self, uuid: Uuid, primary_key: Option, ret: oneshot::Sender>) { + async fn handle_create_index( + &self, + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + ) { let result = self.store.create_index(uuid, primary_key).await; let _ = ret.send(result); } - async fn handle_update(&self, meta: Processing, data: File, ret: oneshot::Sender) { + async fn handle_update( + &self, + meta: Processing, + data: File, + ret: oneshot::Sender, + ) { info!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); @@ -105,13 +168,30 @@ impl IndexActor { tokio::task::spawn_blocking(move || { let result = update_handler.handle_update(meta, data, index); let _ = ret.send(result); - }).await; + }) + .await; } async fn handle_settings(&self, uuid: Uuid, ret: oneshot::Sender>) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - let result = index.settings() + let result = index.settings().map_err(|e| IndexError::Error(e)); + let _ = ret.send(result); + }) + .await; + } + + async fn handle_fetch_documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ret: oneshot::Sender>>, + ) { + let index = self.store.get(uuid).await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || { + let result = index.retrieve_documents(offset, limit, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)); let _ = ret.send(result); }).await; @@ -133,9 +213,17 @@ impl IndexActorHandle { Self { sender } } - pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + pub async fn create_index( + &self, + uuid: Uuid, + primary_key: Option, + ) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::CreateIndex { ret, uuid, primary_key }; + let msg = IndexMsg::CreateIndex { + ret, + uuid, + primary_key, + }; let _ = self.sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } @@ -160,6 +248,25 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Documents { + uuid, + ret, + offset, + attributes_to_retrieve, + limit, + }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { @@ -190,29 +297,31 @@ impl IndexStore for MapIndexStore { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path) - .map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; let index = Index(Arc::new(index)); Ok(index) - }).await.expect("thread died"); + }) + .await + .expect("thread died"); - self.index_store.write().await.insert(meta.uuid.clone(), index?); + self.index_store + .write() + .await + .insert(meta.uuid.clone(), index?); Ok(meta) } async fn get_or_create(&self, uuid: Uuid) -> Result { match self.index_store.write().await.entry(uuid.clone()) { - Entry::Vacant(entry) => { - match self.meta_store.write().await.entry(uuid.clone()) { - Entry::Vacant(_) => { - todo!() - } - Entry::Occupied(entry) => { - todo!() - } + Entry::Vacant(entry) => match self.meta_store.write().await.entry(uuid.clone()) { + Entry::Vacant(_) => { + todo!() } - } + Entry::Occupied(entry) => { + todo!() + } + }, Entry::Occupied(entry) => Ok(entry.get().clone()), } } @@ -228,6 +337,10 @@ impl MapIndexStore { root.push("indexes/"); let meta_store = Arc::new(RwLock::new(HashMap::new())); let index_store = Arc::new(RwLock::new(HashMap::new())); - Self { meta_store, index_store, root } + Self { + meta_store, + index_store, + root, + } } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index f1bc922d6..47ce04b96 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -11,7 +11,6 @@ use actix_web::web::Bytes; use actix_web::web::Payload; use anyhow::Context; use chrono::{DateTime, Utc}; -use crate::index::{SearchResult, SearchQuery}; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; @@ -19,6 +18,7 @@ use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; +use crate::index::{SearchResult, SearchQuery, Document}; use crate::index::{UpdateResult, Settings, Facets}; pub type UpdateStatus = updates::UpdateStatus; @@ -157,6 +157,21 @@ impl IndexController { Ok(settings) } + pub async fn documents( + &self, + index: String, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> anyhow::Result> { + let uuid = self.uuid_resolver + .resolve(index.clone()) + .await? + .with_context(|| format!("Index {:?} doesn't exist", index))?; + let documents = self.index_handle.documents(uuid, offset, limit, attributes_to_retrieve).await?; + Ok(documents) + } + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { todo!() } diff --git a/src/routes/document.rs b/src/routes/document.rs index 00d037359..af9efc701 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -55,17 +55,18 @@ async fn get_document( data: web::Data, path: web::Path, ) -> Result { - let index = path.index_uid.clone(); - let id = path.document_id.clone(); - match data.retrieve_document(index, id, None as Option>).await { - Ok(document) => { - let json = serde_json::to_string(&document).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + todo!() + //let index = path.index_uid.clone(); + //let id = path.document_id.clone(); + //match data.retrieve_document(index, id, None as Option>).await { + //Ok(document) => { + //let json = serde_json::to_string(&document).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } #[delete( From 581dcd5735ed6374edeed3e27a2eb2472d01e991 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 15:09:00 +0100 Subject: [PATCH 119/522] implement retrieve one document --- src/data/search.rs | 49 +++-------------------------- src/index/mod.rs | 39 +++++++++++++++++++++++ src/index_controller/index_actor.rs | 42 +++++++++++++++++++++++-- src/index_controller/mod.rs | 14 +++++++++ src/routes/document.rs | 23 +++++++------- 5 files changed, 109 insertions(+), 58 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index 319f8b973..753083b7e 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -22,52 +22,13 @@ impl Data { self.index_controller.documents(index, offset, limit, attributes_to_retrieve).await } - pub async fn retrieve_document( + pub async fn retrieve_document( &self, - _index: impl AsRef + Sync + Send + 'static, - _document_id: impl AsRef + Sync + Send + 'static, - _attributes_to_retrieve: Option>, + index: impl AsRef + Sync + Send + 'static, + document_id: impl AsRef + Sync + Send + 'static, + attributes_to_retrieve: Option>, ) -> anyhow::Result> { - todo!() - //let index_controller = self.index_controller.clone(); - //let document: anyhow::Result<_> = tokio::task::spawn_blocking(move || { - //let index = index_controller - //.index(&index)? - //.with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - //let txn = index.read_txn()?; - - //let fields_ids_map = index.fields_ids_map(&txn)?; - - //let attributes_to_retrieve_ids = match attributes_to_retrieve { - //Some(attrs) => attrs - //.iter() - //.filter_map(|f| fields_ids_map.id(f.as_ref())) - //.collect::>(), - //None => fields_ids_map.iter().map(|(id, _)| id).collect(), - //}; - - //let internal_id = index - //.external_documents_ids(&txn)? - //.get(document_id.as_ref().as_bytes()) - //.with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; - - //let document = index - //.documents(&txn, std::iter::once(internal_id))? - //.into_iter() - //.next() - //.map(|(_, d)| d); - - //match document { - //Some(document) => Ok(obkv_to_json( - //&attributes_to_retrieve_ids, - //&fields_ids_map, - //document, - //)?), - //None => bail!("Document with id {} not found", document_id.as_ref()), - //} - //}) - //.await?; - //document + self.index_controller.document(index.as_ref().to_string(), document_id.as_ref().to_string(), attributes_to_retrieve).await } } diff --git a/src/index/mod.rs b/src/index/mod.rs index 8ae2ba6a2..c50c2873c 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -4,6 +4,7 @@ mod updates; use std::sync::Arc; use std::ops::Deref; +use anyhow::{bail, Context}; use serde_json::{Value, Map}; use milli::obkv_to_json; @@ -84,4 +85,42 @@ impl Index { Ok(documents) } + + pub fn retrieve_document>( + &self, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> anyhow::Result> { + let txn = self.read_txn()?; + + let fields_ids_map = self.fields_ids_map(&txn)?; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let internal_id = self + .external_documents_ids(&txn)? + .get(doc_id.as_bytes()) + .with_context(|| format!("Document with id {} not found", doc_id))?; + + let document = self + .documents(&txn, std::iter::once(internal_id))? + .into_iter() + .next() + .map(|(_, d)| d); + + match document { + Some(document) => Ok(obkv_to_json( + &attributes_to_retrieve_ids, + &fields_ids_map, + document, + )?), + None => bail!("Document with id {} not found", doc_id), + } + } } diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 8e59227ac..59f000575 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -8,7 +8,6 @@ use chrono::Utc; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; -use serde_json::{Map, Value}; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; @@ -51,8 +50,14 @@ enum IndexMsg { attributes_to_retrieve: Option>, offset: usize, limit: usize, - ret: oneshot::Sender>>>, + ret: oneshot::Sender>>, }, + Document { + uuid: Uuid, + attributes_to_retrieve: Option>, + doc_id: String, + ret: oneshot::Sender>, + } } struct IndexActor { @@ -126,6 +131,7 @@ impl IndexActor { self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) .await } + Document { uuid, attributes_to_retrieve, doc_id, ret } => self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret).await, } }); @@ -196,6 +202,21 @@ impl IndexActor { let _ = ret.send(result); }).await; } + + async fn handle_fetch_document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ret: oneshot::Sender>, + ) { + let index = self.store.get(uuid).await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || { + let result = index.retrieve_document(doc_id, attributes_to_retrieve) + .map_err(|e| IndexError::Error(e)); + let _ = ret.send(result); + }).await; + } } #[derive(Clone)] @@ -267,6 +288,23 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Document { + uuid, + ret, + doc_id, + attributes_to_retrieve, + }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 47ce04b96..dabf1b936 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -172,6 +172,20 @@ impl IndexController { Ok(documents) } + pub async fn document( + &self, + index: String, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> anyhow::Result { + let uuid = self.uuid_resolver + .resolve(index.clone()) + .await? + .with_context(|| format!("Index {:?} doesn't exist", index))?; + let document = self.index_handle.document(uuid, doc_id, attributes_to_retrieve).await?; + Ok(document) + } + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { todo!() } diff --git a/src/routes/document.rs b/src/routes/document.rs index af9efc701..00d037359 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -55,18 +55,17 @@ async fn get_document( data: web::Data, path: web::Path, ) -> Result { - todo!() - //let index = path.index_uid.clone(); - //let id = path.document_id.clone(); - //match data.retrieve_document(index, id, None as Option>).await { - //Ok(document) => { - //let json = serde_json::to_string(&document).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + let index = path.index_uid.clone(); + let id = path.document_id.clone(); + match data.retrieve_document(index, id, None as Option>).await { + Ok(document) => { + let json = serde_json::to_string(&document).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } #[delete( From 181eaf95f5db0e928308bb36a2777e3198adb070 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 15:10:58 +0100 Subject: [PATCH 120/522] restore update documents --- src/routes/document.rs | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/routes/document.rs b/src/routes/document.rs index 00d037359..965b669d9 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -199,26 +199,25 @@ async fn update_documents( params: web::Query, body: web::Payload, ) -> Result { - todo!() - //let addition_result = data - //.add_documents( - //path.into_inner().index_uid, - //IndexDocumentsMethod::UpdateDocuments, - //UpdateFormat::Json, - //body, - //params.primary_key.clone(), - //).await; + let addition_result = data + .add_documents( + path.into_inner().index_uid, + IndexDocumentsMethod::UpdateDocuments, + UpdateFormat::Json, + body, + params.primary_key.clone(), + ).await; - //match addition_result { - //Ok(update) => { - //let value = serde_json::to_string(&update).unwrap(); - //let response = HttpResponse::Ok().body(value); - //Ok(response) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + match addition_result { + Ok(update) => { + let value = serde_json::to_string(&update).unwrap(); + let response = HttpResponse::Ok().body(value); + Ok(response) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } #[post( From ae5581d37c6d8e1ed025fc3935179f49de7e8dfc Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 15:59:18 +0100 Subject: [PATCH 121/522] implement delete documents --- src/data/updates.rs | 10 ++++------ src/index_controller/mod.rs | 18 ++++++++++++++---- src/routes/document.rs | 27 +++++++++++++-------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 92bba693c..775734ac1 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -44,13 +44,11 @@ impl Data { pub async fn delete_documents( &self, - _index: impl AsRef + Sync + Send + 'static, - _document_ids: Vec, + index: impl AsRef + Sync + Send + 'static, + document_ids: Vec, ) -> anyhow::Result { - todo!() - //let index_controller = self.index_controller.clone(); - //let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; - //Ok(update.into()) + let update = self.index_controller.delete_documents(index.as_ref().to_string(), document_ids).await?; + Ok(update.into()) } pub async fn delete_index( diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index dabf1b936..5f00c2540 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -7,8 +7,7 @@ mod update_handler; use std::path::Path; -use actix_web::web::Bytes; -use actix_web::web::Payload; +use actix_web::web::{Bytes, Payload}; use anyhow::Context; use chrono::{DateTime, Utc}; use futures::stream::StreamExt; @@ -111,8 +110,19 @@ impl IndexController { todo!() } - fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { - todo!() + pub async fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(index).await.unwrap().unwrap(); + let meta = UpdateMeta::DeleteDocuments; + let (sender, receiver) = mpsc::channel(10); + + tokio::task::spawn(async move { + let json = serde_json::to_vec(&document_ids).unwrap(); + let bytes = Bytes::from(json); + let _ = sender.send(Ok(bytes)).await; + }); + + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) } pub async fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { diff --git a/src/routes/document.rs b/src/routes/document.rs index 965b669d9..a848ddcc4 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -229,21 +229,20 @@ async fn delete_documents( path: web::Path, body: web::Json>, ) -> Result { - todo!() - //let ids = body - //.iter() - //.map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) - //.collect(); + let ids = body + .iter() + .map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) + .collect(); - //match data.delete_documents(path.index_uid.clone(), ids).await { - //Ok(result) => { - //let json = serde_json::to_string(&result).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + match data.delete_documents(path.index_uid.clone(), ids).await { + Ok(result) => { + let json = serde_json::to_string(&result).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] From a955e04ab6c0b48821f01011d88409e0214a97c4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 16:04:12 +0100 Subject: [PATCH 122/522] implement clear documents --- src/data/updates.rs | 8 +++----- src/index_controller/mod.rs | 8 ++++++-- src/routes/document.rs | 19 +++++++++---------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 775734ac1..a712f4cc6 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -34,12 +34,10 @@ impl Data { pub async fn clear_documents( &self, - _index: impl AsRef + Sync + Send + 'static, + index: impl AsRef + Sync + Send + 'static, ) -> anyhow::Result { - todo!() - //let index_controller = self.index_controller.clone(); - //let update = tokio::task::spawn_blocking(move || index_controller.clear_documents(index)).await??; - //Ok(update.into()) + let update = self.index_controller.clear_documents(index.as_ref().to_string()).await?; + Ok(update) } pub async fn delete_documents( diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 5f00c2540..d790d1d58 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -106,8 +106,12 @@ impl IndexController { Ok(status) } - fn clear_documents(&self, index: String) -> anyhow::Result { - todo!() + pub async fn clear_documents(&self, index: String) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(index).await.unwrap().unwrap(); + let meta = UpdateMeta::ClearDocuments; + let (_, receiver) = mpsc::channel(1); + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) } pub async fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { diff --git a/src/routes/document.rs b/src/routes/document.rs index a848ddcc4..43d88e4b9 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -250,14 +250,13 @@ async fn clear_all_documents( data: web::Data, path: web::Path, ) -> Result { - todo!() - //match data.clear_documents(path.index_uid.clone()).await { - //Ok(update) => { - //let json = serde_json::to_string(&update).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + match data.clear_documents(path.index_uid.clone()).await { + Ok(update) => { + let json = serde_json::to_string(&update).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } From 6a0a9fec6b5e5bad8cdbaa0fcc52115536ed4869 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 17:25:02 +0100 Subject: [PATCH 123/522] async update store --- src/index_controller/update_actor.rs | 18 ++-- src/index_controller/update_store.rs | 138 +++++++++++++++++---------- 2 files changed, 99 insertions(+), 57 deletions(-) diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index 384c52098..2f3c058bc 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -24,15 +24,15 @@ pub enum UpdateError { } enum UpdateMsg { - CreateIndex{ - uuid: Uuid, - ret: oneshot::Sender>, - }, Update { uuid: Uuid, meta: UpdateMeta, data: mpsc::Receiver>, ret: oneshot::Sender> + }, + ListUpdates { + uuid: Uuid, + ret: oneshot::Sender>>, } } @@ -58,12 +58,14 @@ where D: AsRef<[u8]> + Sized + 'static, } async fn run(mut self) { + use UpdateMsg::*; + info!("started update actor."); loop { match self.inbox.recv().await { - Some(UpdateMsg::Update { uuid, meta, data, ret }) => self.handle_update(uuid, meta, data, ret).await, - Some(_) => {} + Some(Update { uuid, meta, data, ret }) => self.handle_update(uuid, meta, data, ret).await, + Some(ListUpdates { uuid, ret }) => self.handle_list_updates(uuid, ret).await, None => {} } } @@ -99,6 +101,10 @@ where D: AsRef<[u8]> + Sized + 'static, let _ = ret.send(result); }).await; } + + async fn handle_list_updates(&self, uuid: Uuid, ret: oneshot::Sender>>) { + todo!() + } } #[derive(Clone)] diff --git a/src/index_controller/update_store.rs b/src/index_controller/update_store.rs index ae4bfb8d8..02c15ed8c 100644 --- a/src/index_controller/update_store.rs +++ b/src/index_controller/update_store.rs @@ -1,12 +1,12 @@ +use std::fs::remove_file; use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; -use std::fs::remove_file; -use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson}; -use heed::{EnvOpenOptions, Env, Database}; -use serde::{Serialize, Deserialize}; +use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; +use heed::{Database, Env, EnvOpenOptions}; +use serde::{Deserialize, Serialize}; use std::fs::File; +use tokio::sync::mpsc; use uuid::Uuid; use crate::index_controller::updates::*; @@ -22,17 +22,26 @@ pub struct UpdateStore { failed_meta: Database, SerdeJson>>, aborted_meta: Database, SerdeJson>>, processing: Arc>>>, - notification_sender: Sender<()>, + notification_sender: mpsc::Sender<()>, } pub trait HandleUpdate { - fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed>; + fn handle_update( + &mut self, + meta: Processing, + content: File, + ) -> Result, Failed>; } impl HandleUpdate for F -where F: FnMut(Processing, File) -> Result, Failed> +where + F: FnMut(Processing, File) -> Result, Failed>, { - fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed> { + fn handle_update( + &mut self, + meta: Processing, + content: File, + ) -> Result, Failed> { self(meta, content) } } @@ -46,11 +55,11 @@ where pub fn open( mut options: EnvOpenOptions, path: P, - mut update_handler: U, + update_handler: U, ) -> heed::Result> where P: AsRef, - U: HandleUpdate + Send + 'static, + U: HandleUpdate + Sync + Clone + Send + 'static, { options.max_dbs(5); @@ -62,7 +71,7 @@ where let failed_meta = env.create_database(Some("failed-meta"))?; let processing = Arc::new(RwLock::new(None)); - let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + let (notification_sender, mut notification_receiver) = mpsc::channel(10); // Send a first notification to trigger the process. let _ = notification_sender.send(()); @@ -80,13 +89,19 @@ where // We need a weak reference so we can take ownership on the arc later when we // want to close the index. let update_store_weak = Arc::downgrade(&update_store); - std::thread::spawn(move || { + tokio::task::spawn(async move { // Block and wait for something to process. - 'outer: for _ in notification_receiver { + 'outer: while let Some(_) = notification_receiver.recv().await { loop { match update_store_weak.upgrade() { Some(update_store) => { - match update_store.process_pending_update(&mut update_handler) { + let handler = update_handler.clone(); + let res = tokio::task::spawn_blocking(move || { + update_store.process_pending_update(handler) + }) + .await + .unwrap(); + match res { Ok(Some(_)) => (), Ok(None) => break, Err(e) => eprintln!("error while processing update: {}", e), @@ -108,17 +123,20 @@ where /// Returns the new biggest id to use to store the new update. fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { - let last_pending = self.pending_meta + let last_pending = self + .pending_meta .remap_data_type::() .last(txn)? .map(|(k, _)| k.get()); - let last_processed = self.processed_meta + let last_processed = self + .processed_meta .remap_data_type::() .last(txn)? .map(|(k, _)| k.get()); - let last_aborted = self.aborted_meta + let last_aborted = self + .aborted_meta .remap_data_type::() .last(txn)? .map(|(k, _)| k.get()); @@ -154,21 +172,22 @@ where let meta = Pending::new(meta, update_id, index_uuid); self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, &content.as_ref().to_owned())?; + self.pending + .put(&mut wtxn, &update_key, &content.as_ref().to_owned())?; wtxn.commit()?; - if let Err(e) = self.notification_sender.try_send(()) { - assert!(!e.is_disconnected(), "update notification channel is disconnected"); - } + self.notification_sender + .blocking_send(()) + .expect("Update store loop exited."); Ok(meta) } /// Executes the user provided function on the next pending update (the one with the lowest id). /// This is asynchronous as it let the user process the update with a read-only txn and /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, handler: &mut U) -> heed::Result> + fn process_pending_update(&self, mut handler: U) -> heed::Result> where - U: HandleUpdate + Send + 'static, + U: HandleUpdate, { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; @@ -178,7 +197,8 @@ where // a reader while processing it, not a writer. match first_meta { Some((first_id, pending)) => { - let content_path = self.pending + let content_path = self + .pending .get(&rtxn, &first_id)? .expect("associated update content"); @@ -186,10 +206,7 @@ where // to the update handler. Processing store is non persistent to be able recover // from a failure let processing = pending.processing(); - self.processing - .write() - .unwrap() - .replace(processing.clone()); + self.processing.write().unwrap().replace(processing.clone()); let file = File::open(&content_path)?; // Process the pending update using the provided user function. let result = handler.handle_update(processing, file); @@ -199,10 +216,7 @@ where // we must remove the content from the pending and processing stores and // write the *new* meta to the processed-meta store and commit. let mut wtxn = self.env.write_txn()?; - self.processing - .write() - .unwrap() - .take(); + self.processing.write().unwrap().take(); self.pending_meta.delete(&mut wtxn, &first_id)?; remove_file(&content_path)?; self.pending.delete(&mut wtxn, &first_id)?; @@ -213,8 +227,8 @@ where wtxn.commit()?; Ok(Some(())) - }, - None => Ok(None) + } + None => Ok(None), } } @@ -241,7 +255,13 @@ where let failed_iter = self.failed_meta.iter(&rtxn)?; // We execute the user defined function with both iterators. - (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + (f)( + processing, + processed_iter, + aborted_iter, + pending_iter, + failed_iter, + ) } /// Returns the update associated meta or `None` if the update doesn't exist. @@ -340,22 +360,33 @@ mod tests { use std::time::{Duration, Instant}; impl HandleUpdate for F - where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { - fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { - self(meta, content) - } + where + F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static, + { + fn handle_update( + &mut self, + meta: Processing, + content: &[u8], + ) -> Result, Failed> { + self(meta, content) } + } #[test] fn simple() { let dir = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); + let update_store = UpdateStore::open( + options, + dir, + |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }, + ) + .unwrap(); let meta = String::from("kiki"); let update = update_store.register_update(meta, &[]).unwrap(); @@ -374,12 +405,17 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); + let update_store = UpdateStore::open( + options, + dir, + |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }, + ) + .unwrap(); let before_register = Instant::now(); From f090f42e7a250a7a94ef094a98d4fba19c2c2d34 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 19:18:01 +0100 Subject: [PATCH 124/522] multi index store create two channels for Index handler, one for writes and one for reads, so write are processed one at a time, while reads are processed in parallel. --- Cargo.lock | 4 +- src/index_controller/index_actor.rs | 129 ++++++++++++++++--------- src/index_controller/update_actor.rs | 138 +++++++++++++++++++-------- 3 files changed, 188 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f3b20e89..5f1134379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" version = "0.3.0" @@ -428,7 +430,7 @@ checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "assert-json-diff" version = "1.0.1" -source = "git+https://github.com/qdequele/assert-json-diff#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" dependencies = [ "serde", "serde_json", diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 59f000575..0e74d6665 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -11,6 +11,8 @@ use log::info; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; +use std::future::Future; +use futures::pin_mut; use super::update_handler::UpdateHandler; use crate::index::UpdateResult as UResult; @@ -61,7 +63,8 @@ enum IndexMsg { } struct IndexActor { - inbox: Option>, + read_receiver: Option>, + write_receiver: Option>, update_handler: Arc, store: S, } @@ -82,60 +85,96 @@ trait IndexStore { } impl IndexActor { - fn new(inbox: mpsc::Receiver, store: S) -> Self { + fn new( + read_receiver: mpsc::Receiver, + write_receiver: mpsc::Receiver, + store: S + ) -> Self { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).unwrap(); let update_handler = Arc::new(update_handler); - let inbox = Some(inbox); + let read_receiver = Some(read_receiver); + let write_receiver = Some(write_receiver); Self { - inbox, + read_receiver, + write_receiver, store, update_handler, } } + /// `run` poll the write_receiver and read_receiver concurrently, but while messages send + /// through the read channel are processed concurrently, the messages sent through the write + /// channel are processed one at a time. async fn run(mut self) { - use IndexMsg::*; - - let mut inbox = self - .inbox + let mut read_receiver = self + .read_receiver .take() .expect("Index Actor must have a inbox at this point."); - let stream = stream! { + let read_stream = stream! { loop { - match inbox.recv().await { + match read_receiver.recv().await { Some(msg) => yield msg, None => break, } } }; - let fut = stream.for_each_concurrent(Some(10), |msg| async { - match msg { - CreateIndex { - uuid, - primary_key, - ret, - } => self.handle_create_index(uuid, primary_key, ret).await, - Update { ret, meta, data } => self.handle_update(meta, data, ret).await, - Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, - Settings { ret, uuid } => self.handle_settings(uuid, ret).await, - Documents { - ret, - uuid, - attributes_to_retrieve, - offset, - limit, - } => { - self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) - .await - } - Document { uuid, attributes_to_retrieve, doc_id, ret } => self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret).await, - } - }); + let mut write_receiver = self + .write_receiver + .take() + .expect("Index Actor must have a inbox at this point."); - fut.await; + let write_stream = stream! { + loop { + match write_receiver.recv().await { + Some(msg) => yield msg, + None => break, + } + } + }; + + pin_mut!(write_stream); + pin_mut!(read_stream); + + let fut1 = read_stream.for_each_concurrent(Some(10), |msg| self.handle_message(msg)); + let fut2 = write_stream.for_each_concurrent(Some(1), |msg| self.handle_message(msg)); + + let fut1: Box + Unpin + Send> = Box::new(fut1); + let fut2: Box + Unpin + Send> = Box::new(fut2); + + //let futures = futures::stream::futures_unordered::FuturesUnordered::new(); + //futures.push(fut1); + //futures.push(fut2); + //futures.for_each(f) + tokio::join!(fut1, fut2); + + } + + async fn handle_message(&self, msg: IndexMsg) { + use IndexMsg::*; + match msg { + CreateIndex { + uuid, + primary_key, + ret, + } => self.handle_create_index(uuid, primary_key, ret).await, + Update { ret, meta, data } => self.handle_update(meta, data, ret).await, + Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, + Settings { ret, uuid } => self.handle_settings(uuid, ret).await, + Documents { + ret, + uuid, + attributes_to_retrieve, + offset, + limit, + } => { + self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) + .await + } + Document { uuid, attributes_to_retrieve, doc_id, ret } => self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret).await, + } } async fn handle_search( @@ -221,17 +260,19 @@ impl IndexActor { #[derive(Clone)] pub struct IndexActorHandle { - sender: mpsc::Sender, + read_sender: mpsc::Sender, + write_sender: mpsc::Sender, } impl IndexActorHandle { pub fn new(path: impl AsRef) -> Self { - let (sender, receiver) = mpsc::channel(100); + let (read_sender, read_receiver) = mpsc::channel(100); + let (write_sender, write_receiver) = mpsc::channel(100); let store = MapIndexStore::new(path); - let actor = IndexActor::new(receiver, store); + let actor = IndexActor::new(read_receiver, write_receiver, store); tokio::task::spawn(actor.run()); - Self { sender } + Self { read_sender, write_sender } } pub async fn create_index( @@ -245,28 +286,28 @@ impl IndexActorHandle { uuid, primary_key, }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } pub async fn update(&self, meta: Processing, data: std::fs::File) -> UpdateResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Update { ret, meta, data }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } pub async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Search { uuid, query, ret }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } pub async fn settings(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Settings { uuid, ret }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -285,7 +326,7 @@ impl IndexActorHandle { attributes_to_retrieve, limit, }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -302,7 +343,7 @@ impl IndexActorHandle { doc_id, attributes_to_retrieve, }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } } diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index 2f3c058bc..a9e2a412a 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -1,17 +1,18 @@ use std::fs::create_dir_all; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::collections::{HashMap, hash_map::Entry}; -use log::info; use super::index_actor::IndexActorHandle; +use log::info; use thiserror::Error; -use tokio::sync::{mpsc, oneshot}; -use uuid::Uuid; use tokio::fs::File; use tokio::io::AsyncWriteExt; +use tokio::sync::{mpsc, oneshot, RwLock}; +use uuid::Uuid; -use crate::index_controller::{UpdateMeta, UpdateStatus}; use crate::index::UpdateResult; +use crate::index_controller::{UpdateMeta, UpdateStatus}; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; @@ -28,33 +29,42 @@ enum UpdateMsg { uuid: Uuid, meta: UpdateMeta, data: mpsc::Receiver>, - ret: oneshot::Sender> + ret: oneshot::Sender>, }, ListUpdates { uuid: Uuid, ret: oneshot::Sender>>, - } + }, } -struct UpdateActor { +struct UpdateActor { path: PathBuf, - store: Arc, + store: S, inbox: mpsc::Receiver>, - index_handle: IndexActorHandle, } -impl UpdateActor -where D: AsRef<[u8]> + Sized + 'static, +#[async_trait::async_trait] +trait UpdateStoreStore { + async fn get_or_create(&self, uuid: Uuid) -> Result>; +} + +impl UpdateActor +where + D: AsRef<[u8]> + Sized + 'static, + S: UpdateStoreStore, { fn new( - store: Arc, + store: S, inbox: mpsc::Receiver>, - index_handle: IndexActorHandle, path: impl AsRef, - ) -> Self { + ) -> Self { let path = path.as_ref().to_owned().join("update_files"); create_dir_all(&path).unwrap(); - Self { store, inbox, index_handle, path } + Self { + store, + inbox, + path, + } } async fn run(mut self) { @@ -64,15 +74,26 @@ where D: AsRef<[u8]> + Sized + 'static, loop { match self.inbox.recv().await { - Some(Update { uuid, meta, data, ret }) => self.handle_update(uuid, meta, data, ret).await, + Some(Update { + uuid, + meta, + data, + ret, + }) => self.handle_update(uuid, meta, data, ret).await, Some(ListUpdates { uuid, ret }) => self.handle_list_updates(uuid, ret).await, None => {} } } } - async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver>, ret: oneshot::Sender>) { - let store = self.store.clone(); + async fn handle_update( + &self, + uuid: Uuid, + meta: UpdateMeta, + mut payload: mpsc::Receiver>, + ret: oneshot::Sender>, + ) { + let update_store = self.store.get_or_create(uuid).await.unwrap(); let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); let mut file = File::create(&path).await.unwrap(); @@ -84,7 +105,7 @@ where D: AsRef<[u8]> + Sized + 'static, } Err(e) => { ret.send(Err(UpdateError::Error(e))); - return + return; } } } @@ -94,15 +115,20 @@ where D: AsRef<[u8]> + Sized + 'static, let file = file.into_std().await; let result = tokio::task::spawn_blocking(move || { - let result = store + let result = update_store .register_update(meta, path, uuid) .map(|pending| UpdateStatus::Pending(pending)) .map_err(|e| UpdateError::Error(Box::new(e))); let _ = ret.send(result); - }).await; + }) + .await; } - async fn handle_list_updates(&self, uuid: Uuid, ret: oneshot::Sender>>) { + async fn handle_list_updates( + &self, + uuid: Uuid, + ret: oneshot::Sender>>, + ) { todo!() } } @@ -113,29 +139,26 @@ pub struct UpdateActorHandle { } impl UpdateActorHandle -where D: AsRef<[u8]> + Sized + 'static, +where + D: AsRef<[u8]> + Sized + 'static, { pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { + let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); - let mut options = heed::EnvOpenOptions::new(); - options.map_size(4096 * 100_000); + let store = MapUpdateStoreStore::new(index_handle, &path); + let actor = UpdateActor::new(store, receiver, path); - let path = path - .as_ref() - .to_owned() - .join("updates"); - - create_dir_all(&path).unwrap(); - let index_handle_clone = index_handle.clone(); - let store = UpdateStore::open(options, &path, move |meta, file| { - futures::executor::block_on(index_handle_clone.update(meta, file)) - }).unwrap(); - let actor = UpdateActor::new(store, receiver, index_handle, path); tokio::task::spawn_local(actor.run()); + Self { sender } } - pub async fn update(&self, meta: UpdateMeta, data: mpsc::Receiver>, uuid: Uuid) -> Result { + pub async fn update( + &self, + meta: UpdateMeta, + data: mpsc::Receiver>, + uuid: Uuid, + ) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Update { uuid, @@ -143,7 +166,46 @@ where D: AsRef<[u8]> + Sized + 'static, meta, ret, }; - let _ = self.sender.send(msg).await; + let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } } + +struct MapUpdateStoreStore { + db: Arc>>>, + index_handle: IndexActorHandle, + path: PathBuf, +} + +impl MapUpdateStoreStore { + fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { + let db = Arc::new(RwLock::new(HashMap::new())); + let path = path.as_ref().to_owned(); + Self { db, index_handle, path } + } +} + +#[async_trait::async_trait] +impl UpdateStoreStore for MapUpdateStoreStore { + async fn get_or_create(&self, uuid: Uuid) -> Result> { + match self.db.write().await.entry(uuid) { + Entry::Vacant(e) => { + let mut options = heed::EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let path = self.path.clone().join(format!("updates-{}", e.key())); + create_dir_all(&path).unwrap(); + let index_handle = self.index_handle.clone(); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle.update(meta, file)) + }).unwrap(); + let store = e.insert(store); + Ok(store.clone()) + } + Entry::Occupied(e) => { + Ok(e.get().clone()) + } + } + } +} + + From c2282ab5cb0d94dedb04978f2931c4078dbe3fee Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 19:30:13 +0100 Subject: [PATCH 125/522] non local udpate actor --- src/index_controller/update_actor.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index a9e2a412a..4d1ff78fb 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -140,7 +140,7 @@ pub struct UpdateActorHandle { impl UpdateActorHandle where - D: AsRef<[u8]> + Sized + 'static, + D: AsRef<[u8]> + Sized + 'static + Sync + Send, { pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { let path = path.as_ref().to_owned().join("updates"); @@ -148,7 +148,7 @@ where let store = MapUpdateStoreStore::new(index_handle, &path); let actor = UpdateActor::new(store, receiver, path); - tokio::task::spawn_local(actor.run()); + tokio::task::spawn(actor.run()); Self { sender } } @@ -207,5 +207,3 @@ impl UpdateStoreStore for MapUpdateStoreStore { } } } - - From a9c7b7374434c31b0f1977445f72dd4a2c104b6b Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 5 Mar 2021 18:34:04 +0100 Subject: [PATCH 126/522] implement list all updates --- src/data/updates.rs | 5 +- src/index_controller/mod.rs | 10 ++- src/index_controller/update_actor.rs | 103 ++++++++++++++++++--------- src/routes/index.rs | 2 +- 4 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index a712f4cc6..3e3837861 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -65,9 +65,8 @@ impl Data { //self.index_controller.update_status(index, uid) } - pub fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { - todo!() - //self.index_controller.all_update_status(index) + pub async fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { + self.index_controller.all_update_status(index.as_ref().to_string()).await } pub fn update_index( diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index d790d1d58..8eb1684a2 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -107,7 +107,7 @@ impl IndexController { } pub async fn clear_documents(&self, index: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await.unwrap().unwrap(); + let uuid = self.uuid_resolver.resolve(index).await?.unwrap(); let meta = UpdateMeta::ClearDocuments; let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; @@ -154,8 +154,12 @@ impl IndexController { todo!() } - fn all_update_status(&self, index: String) -> anyhow::Result> { - todo!() + pub async fn all_update_status(&self, index: String) -> anyhow::Result> { + let uuid = self.uuid_resolver + .resolve(index).await? + .context("index not found")?; + let result = self.update_handle.get_all_updates_status(uuid).await?; + Ok(result) } pub fn list_indexes(&self) -> anyhow::Result> { diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index 4d1ff78fb..b236df2a6 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -1,7 +1,7 @@ +use std::collections::{hash_map::Entry, HashMap}; use std::fs::create_dir_all; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::collections::{HashMap, hash_map::Entry}; use super::index_actor::IndexActorHandle; use log::info; @@ -10,6 +10,7 @@ use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; +use itertools::Itertools; use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; @@ -22,6 +23,8 @@ type PayloadData = std::result::Result), + #[error("Index {0} doesn't exist.")] + UnexistingIndex(Uuid), } enum UpdateMsg { @@ -46,6 +49,7 @@ struct UpdateActor { #[async_trait::async_trait] trait UpdateStoreStore { async fn get_or_create(&self, uuid: Uuid) -> Result>; + async fn get(&self, uuid: &Uuid) -> Result>>; } impl UpdateActor @@ -53,24 +57,16 @@ where D: AsRef<[u8]> + Sized + 'static, S: UpdateStoreStore, { - fn new( - store: S, - inbox: mpsc::Receiver>, - path: impl AsRef, - ) -> Self { + fn new(store: S, inbox: mpsc::Receiver>, path: impl AsRef) -> Self { let path = path.as_ref().to_owned().join("update_files"); create_dir_all(&path).unwrap(); - Self { - store, - inbox, - path, - } + Self { store, inbox, path } } async fn run(mut self) { use UpdateMsg::*; - info!("started update actor."); + info!("Started update actor."); loop { match self.inbox.recv().await { @@ -79,8 +75,12 @@ where meta, data, ret, - }) => self.handle_update(uuid, meta, data, ret).await, - Some(ListUpdates { uuid, ret }) => self.handle_list_updates(uuid, ret).await, + }) => { + let _ = ret.send(self.handle_update(uuid, meta, data).await); + } + Some(ListUpdates { uuid, ret }) => { + let _ = ret.send(self.handle_list_updates(uuid).await); + } , None => {} } } @@ -91,45 +91,68 @@ where uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver>, - ret: oneshot::Sender>, - ) { - let update_store = self.store.get_or_create(uuid).await.unwrap(); + ) -> Result { + let update_store = self.store.get_or_create(uuid).await?; let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); - let mut file = File::create(&path).await.unwrap(); + let mut file = File::create(&path).await + .map_err(|e| UpdateError::Error(Box::new(e)))?; while let Some(bytes) = payload.recv().await { match bytes { Ok(bytes) => { - file.write_all(bytes.as_ref()).await; + file.write_all(bytes.as_ref()).await + .map_err(|e| UpdateError::Error(Box::new(e)))?; } Err(e) => { - ret.send(Err(UpdateError::Error(e))); - return; + return Err(UpdateError::Error(e)); } } } - file.flush().await; + file.flush().await + .map_err(|e| UpdateError::Error(Box::new(e)))?; let file = file.into_std().await; - let result = tokio::task::spawn_blocking(move || { + tokio::task::spawn_blocking(move || { let result = update_store .register_update(meta, path, uuid) .map(|pending| UpdateStatus::Pending(pending)) .map_err(|e| UpdateError::Error(Box::new(e))); - let _ = ret.send(result); + result }) - .await; + .await + .map_err(|e| UpdateError::Error(Box::new(e)))? } async fn handle_list_updates( &self, uuid: Uuid, - ret: oneshot::Sender>>, - ) { - todo!() + ) -> Result> { + let store = self.store.get(&uuid).await?; + tokio::task::spawn_blocking(move || { + let result = match store { + Some(update_store) => { + let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { + Ok(processing + .map(UpdateStatus::from) + .into_iter() + .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) + .chain(aborted.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .chain(processed.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .chain(failed.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .sorted_by(|a, b| a.id().cmp(&b.id())) + .collect()) + }) + .map_err(|e| UpdateError::Error(Box::new(e)))?; + Ok(updates) + } + None => Err(UpdateError::UnexistingIndex(uuid)), + }; + result + }).await + .map_err(|e| UpdateError::Error(Box::new(e)))? } } @@ -169,6 +192,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::ListUpdates { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { @@ -181,7 +211,11 @@ impl MapUpdateStoreStore { fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { let db = Arc::new(RwLock::new(HashMap::new())); let path = path.as_ref().to_owned(); - Self { db, index_handle, path } + Self { + db, + index_handle, + path, + } } } @@ -197,13 +231,16 @@ impl UpdateStoreStore for MapUpdateStoreStore { let index_handle = self.index_handle.clone(); let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle.update(meta, file)) - }).unwrap(); + }) + .unwrap(); let store = e.insert(store); Ok(store.clone()) } - Entry::Occupied(e) => { - Ok(e.get().clone()) - } + Entry::Occupied(e) => Ok(e.get().clone()), } } + + async fn get(&self, uuid: &Uuid) -> Result>> { + Ok(self.db.read().await.get(uuid).cloned()) + } } diff --git a/src/routes/index.rs b/src/routes/index.rs index 813256517..a8618d9dd 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -154,7 +154,7 @@ async fn get_all_updates_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_updates_status(&path.index_uid); + let result = data.get_updates_status(&path.index_uid).await; match result { Ok(metas) => { let json = serde_json::to_string(&metas).unwrap(); From 4552c42f88b9ccf0132de87540810c960c3e4e06 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 13:33:24 +0100 Subject: [PATCH 127/522] deduplicate pending and processing updates --- .../local_index_controller/mod.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs index 14efe42c7..b4864fcb5 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/mod.rs @@ -103,11 +103,27 @@ impl IndexController for LocalIndexController { fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>> { match self.indexes.index(&index)? { Some((_, update_store)) => { - let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { + let updates = update_store.iter_metas(|processing, processed, aborted, pending, failed| { + let processing_id = processing + .as_ref() + .map(|p| p.id()); + Ok(processing .map(UpdateStatus::from) .into_iter() - .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) + .chain(pending. + filter_map(|p| p.ok()) + // if an update is processing, filter out this update from the pending + // updates. + .filter(|(_, u)| { + println!("processing: {:?}", processing_id); + processing_id + .map(|id| { + println!("id: {}, pending: {}", id, u.id()); + id != u.id() + }) + .unwrap_or(true)}) + .map(|(_, u)| UpdateStatus::from(u))) .chain(aborted.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) .chain(processed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) .chain(failed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) From 7d2ae9089e809a06b28fe1e678a595161f410891 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 13:35:56 +0100 Subject: [PATCH 128/522] restore test --- .../local_index_controller/mod.rs | 15 +++++---------- meilisearch-http/tests/updates/mod.rs | 2 -- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/meilisearch-http/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs index b4864fcb5..6b917b0e1 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/mod.rs @@ -112,17 +112,12 @@ impl IndexController for LocalIndexController { .map(UpdateStatus::from) .into_iter() .chain(pending. - filter_map(|p| p.ok()) - // if an update is processing, filter out this update from the pending + filter_map(Result::ok) + // If an update is processing, filter out this update from the pending // updates. - .filter(|(_, u)| { - println!("processing: {:?}", processing_id); - processing_id - .map(|id| { - println!("id: {}, pending: {}", id, u.id()); - id != u.id() - }) - .unwrap_or(true)}) + .filter(|(_, u)| processing_id + .map(|id| id != u.id()) + .unwrap_or(true)) .map(|(_, u)| UpdateStatus::from(u))) .chain(aborted.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) .chain(processed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 3fff2d911..03b307daf 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -50,9 +50,7 @@ async fn list_no_updates() { assert!(response.as_array().unwrap().is_empty()); } -// TODO: fix #32 #[actix_rt::test] -#[ignore] async fn list_updates() { let server = Server::new().await; let index = server.index("test"); From c0515bcfe206247f49c14bdf8fec346351f9bd54 Mon Sep 17 00:00:00 2001 From: marin Date: Fri, 5 Mar 2021 19:08:28 +0100 Subject: [PATCH 129/522] Update src/index_controller/local_index_controller/mod.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Renault --- .../src/index_controller/local_index_controller/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meilisearch-http/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs index 6b917b0e1..d3fa532dc 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/mod.rs @@ -116,8 +116,7 @@ impl IndexController for LocalIndexController { // If an update is processing, filter out this update from the pending // updates. .filter(|(_, u)| processing_id - .map(|id| id != u.id()) - .unwrap_or(true)) + .map_or(true, |id| id != u.id())) .map(|(_, u)| UpdateStatus::from(u))) .chain(aborted.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) .chain(processed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) From 3987d17e408af6950ee038291c2e14f80e6a5076 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 11:00:15 +0100 Subject: [PATCH 130/522] add indx uid format guard on create ops --- meilisearch-http/src/data/mod.rs | 9 +++++++++ meilisearch-http/src/data/updates.rs | 13 ++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index ed5ce4952..fee29561a 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -8,6 +8,7 @@ use std::ops::Deref; use std::sync::Arc; use sha2::Digest; +use anyhow::bail; use crate::index_controller::{IndexController, LocalIndexController, IndexMetadata, Settings, IndexSettings}; use crate::option::Opt; @@ -126,6 +127,9 @@ impl Data { } pub fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { + if !is_index_uid_valid(name.as_ref()) { + bail!("invalid index uid: {:?}", name.as_ref()) + } let settings = IndexSettings { name: Some(name.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), @@ -145,3 +149,8 @@ impl Data { &self.api_keys } } + +fn is_index_uid_valid(uid: &str) -> bool { + uid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') +} + diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index fbb9be801..a784dce99 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,13 +1,13 @@ use std::ops::Deref; +use anyhow::bail; use async_compression::tokio_02::write::GzipEncoder; use futures_util::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use tokio::io::AsyncWriteExt; -use crate::index_controller::UpdateStatus; -use crate::index_controller::{IndexController, Settings, IndexSettings, IndexMetadata}; -use super::Data; +use super::{Data, is_index_uid_valid}; +use crate::index_controller::{UpdateStatus, IndexController, Settings, IndexSettings, IndexMetadata}; impl Data { pub async fn add_documents( @@ -22,6 +22,10 @@ impl Data { B: Deref, E: std::error::Error + Send + Sync + 'static, { + if !is_index_uid_valid(index.as_ref()) { + bail!("invalid index uid: {:?}", index.as_ref()) + } + let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; let file = tokio::fs::File::from_std(file?); let mut encoder = GzipEncoder::new(file); @@ -57,6 +61,9 @@ impl Data { index: impl AsRef + Send + Sync + 'static, settings: Settings ) -> anyhow::Result { + if !is_index_uid_valid(index.as_ref()) { + bail!("invalid index uid: {:?}", index.as_ref()) + } let index_controller = self.index_controller.clone(); let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; Ok(update.into()) From 561f29042c3d7c3b54211573462f209a02f524e7 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 11:15:48 +0100 Subject: [PATCH 131/522] add tests --- Cargo.lock | 7 +++++++ meilisearch-http/Cargo.toml | 1 + meilisearch-http/tests/common/server.rs | 7 ++++--- .../tests/documents/add_documents.rs | 16 ++++++++++++++++ meilisearch-http/tests/index/create_index.rs | 4 +--- meilisearch-http/tests/settings/get_settings.rs | 9 +++++++++ 6 files changed, 38 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17575cd48..e1b3b0762 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1677,6 +1677,7 @@ dependencies = [ "tempdir", "tempfile", "tokio", + "urlencoding", "uuid", "vergen", ] @@ -3298,6 +3299,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" + [[package]] name = "utf8-width" version = "0.1.4" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index b760a3d27..2fe9d92d7 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -71,6 +71,7 @@ serde_url_params = "0.2.0" tempdir = "0.3.7" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } tokio = { version = "0.2", features = ["macros", "time"] } +urlencoding = "1.1.1" [features] default = ["sentry"] diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index d7d76445c..943284736 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -1,7 +1,8 @@ -use tempdir::TempDir; +use actix_web::http::StatusCode; use byte_unit::{Byte, ByteUnit}; use serde_json::Value; -use actix_web::http::StatusCode; +use tempdir::TempDir; +use urlencoding::encode; use meilisearch_http::data::Data; use meilisearch_http::option::{Opt, IndexerOpts}; @@ -60,7 +61,7 @@ impl Server { /// Returns a view to an index. There is no guarantee that the index exists. pub fn index<'a>(&'a self, uid: impl AsRef) -> Index<'a> { Index { - uid: uid.as_ref().to_string(), + uid: encode(uid.as_ref()), service: &self.service, } } diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 70d6aab68..63724af18 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -45,6 +45,22 @@ async fn add_documents_no_index_creation() { assert_eq!(response["primaryKey"], "id"); } +#[actix_rt::test] +async fn document_add_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (_response, code) = index.add_documents(json!([]), None).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn document_update_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (_response, code) = index.update_documents(json!([]), None).await; + assert_eq!(code, 400); +} + #[actix_rt::test] async fn document_addition_with_primary_key() { let server = Server::new().await; diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 3ff452c33..c26941b91 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -47,12 +47,10 @@ async fn create_existing_index() { assert_eq!(code, 400); } -// test fails (issue #46) #[actix_rt::test] -#[ignore] async fn create_with_invalid_index_uid() { let server = Server::new().await; - let index = server.index("test test"); + let index = server.index("test test#!"); let (_, code) = index.create(None).await; assert_eq!(code, 400); } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index bae044acb..0e4d991da 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -90,6 +90,15 @@ async fn update_setting_unexisting_index() { assert_eq!(code, 200); } +#[actix_rt::test] +async fn update_setting_unexisting_index_invalid_uid() { + let server = Server::new().await; + let index = server.index("test##! "); + let (_response, code) = index.update_settings(json!({})).await; + println!("response: {}", _response); + assert_eq!(code, 400); +} + macro_rules! test_setting_routes { ($($setting:ident), *) => { $( From d52e6fc21eb209ecf74025b68a9ed5dd3fadb647 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 10:42:27 +0100 Subject: [PATCH 132/522] fix settings delete bug --- meilisearch-http/src/data/updates.rs | 5 +++-- .../index_controller/local_index_controller/mod.rs | 11 ++++++++--- meilisearch-http/src/index_controller/mod.rs | 5 +++-- meilisearch-http/src/routes/settings/mod.rs | 8 ++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index fbb9be801..c9c8b2c41 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -55,10 +55,11 @@ impl Data { pub async fn update_settings( &self, index: impl AsRef + Send + Sync + 'static, - settings: Settings + settings: Settings, + create: bool, ) -> anyhow::Result { let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; + let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings, create)).await??; Ok(update.into()) } diff --git a/meilisearch-http/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs index 14efe42c7..7669bbcba 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/mod.rs @@ -5,7 +5,7 @@ mod update_handler; use std::path::Path; use std::sync::Arc; -use anyhow::{bail, Context}; +use anyhow::{bail, Context, anyhow}; use itertools::Itertools; use milli::Index; @@ -51,9 +51,14 @@ impl IndexController for LocalIndexController { fn update_settings>( &self, index: S, - settings: super::Settings + settings: super::Settings, + create: bool, ) -> anyhow::Result> { - let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; + let (_, update_store) = if create { + self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)? + } else { + self.indexes.index(&index)?.ok_or_else(|| anyhow!("Index {:?} doesn't exist", index.as_ref()))? + }; let meta = UpdateMeta::Settings(settings); let pending = update_store.register_update(meta, &[])?; Ok(pending.into()) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index b20e43749..2d23ebadf 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -142,8 +142,9 @@ pub trait IndexController { fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result; /// Updates an index settings. If the index does not exist, it will be created when the update - /// is applied to the index. - fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; + /// is applied to the index. `create` specifies whether an index should be created if not + /// existing. + fn update_settings>(&self, index_uid: S, settings: Settings, create: bool) -> anyhow::Result; /// Create an index with the given `index_uid`. fn create_index(&self, index_settings: IndexSettings) -> Result; diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 00bc4220e..b65729b31 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -26,7 +26,7 @@ macro_rules! make_setting_route { $attr: Some(None), ..Default::default() }; - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, false).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -48,7 +48,7 @@ macro_rules! make_setting_route { ..Default::default() }; - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, true).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -137,7 +137,7 @@ async fn update_all( index_uid: web::Path, body: web::Json, ) -> Result { - match data.update_settings(index_uid.into_inner(), body.into_inner()).await { + match data.update_settings(index_uid.into_inner(), body.into_inner(), true).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -170,7 +170,7 @@ async fn delete_all( index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, false).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) From f04dd2af39fd6c69700ba3c183c2b6b30fd2e4c8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Feb 2021 10:43:50 +0100 Subject: [PATCH 133/522] enable tests delete settings --- meilisearch-http/tests/settings/get_settings.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index bae044acb..427631150 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -51,8 +51,6 @@ async fn test_partial_update() { } #[actix_rt::test] -#[ignore] -// need fix #54 async fn delete_settings_unexisting_index() { let server = Server::new().await; let index = server.index("test"); @@ -123,7 +121,6 @@ macro_rules! test_setting_routes { } #[actix_rt::test] - #[ignore] async fn delete_unexisting_index() { let server = Server::new().await; let url = format!("/indexes/test/settings/{}", From 65ca80bdde170fba76a2d3175b1580ec0654c3cb Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 5 Mar 2021 19:31:49 +0100 Subject: [PATCH 134/522] enable criterion setting --- meilisearch-http/src/data/mod.rs | 8 +++++++- .../local_index_controller/update_handler.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 4 ++-- meilisearch-http/src/routes/settings/mod.rs | 13 +++++++------ 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index ed5ce4952..15db0e4ae 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -106,11 +106,17 @@ impl Data { .map(|(k, v)| (k, v.to_string())) .collect(); + let criteria = index + .criteria(&txn)? + .into_iter() + .map(|v| format!("{:?}", v)) + .collect(); + Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), searchable_attributes: Some(Some(searchable_attributes)), faceted_attributes: Some(Some(faceted_attributes)), - criteria: None, + ranking_rules: Some(Some(criteria)), }) } diff --git a/meilisearch-http/src/index_controller/local_index_controller/update_handler.rs b/meilisearch-http/src/index_controller/local_index_controller/update_handler.rs index 5781a2806..ab2e75206 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/update_handler.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/update_handler.rs @@ -153,7 +153,7 @@ impl UpdateHandler { } // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { + if let Some(ref criteria) = settings.ranking_rules { match criteria { Some(criteria) => builder.set_criteria(criteria.clone()), None => builder.reset_criteria(), diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index b20e43749..fc8efa036 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -83,7 +83,7 @@ pub struct Settings { deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none", )] - pub criteria: Option>>, + pub ranking_rules: Option>>, } impl Settings { @@ -92,7 +92,7 @@ impl Settings { displayed_attributes: Some(None), searchable_attributes: Some(None), faceted_attributes: Some(None), - criteria: Some(None), + ranking_rules: Some(None), } } } diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 00bc4220e..bf6d92133 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -103,11 +103,11 @@ make_setting_route!( //distinct_attribute //); -//make_setting_route!( - //"/indexes/{index_uid}/settings/ranking-rules", - //Vec, - //ranking_rules -//); +make_setting_route!( + "/indexes/{index_uid}/settings/ranking-rules", + Vec, + ranking_rules +); macro_rules! create_services { ($($mod:ident),*) => { @@ -128,7 +128,8 @@ macro_rules! create_services { create_services!( faceted_attributes, displayed_attributes, - searchable_attributes + searchable_attributes, + ranking_rules ); #[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] From f4f42ec4411e954393b5b756deffea096ae516d1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 5 Mar 2021 20:06:10 +0100 Subject: [PATCH 135/522] add tests --- meilisearch-http/src/data/search.rs | 1 + meilisearch-http/tests/settings/get_settings.rs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index f26730fcf..c76e21638 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -62,6 +62,7 @@ impl SearchQuery { documents_ids, found_words, candidates, + .. } = search.execute()?; let mut documents = Vec::new(); diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index bae044acb..6c57863ac 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -8,7 +8,9 @@ async fn get_settings_unexisting_index() { assert_eq!(code, 400) } +// test broken, should be fixed with milli#101 #[actix_rt::test] +#[ignore] async fn get_settings() { let server = Server::new().await; let index = server.index("test"); @@ -16,10 +18,11 @@ async fn get_settings() { let (response, code) = index.settings().await; assert_eq!(code, 200); let settings = response.as_object().unwrap(); - assert_eq!(settings.keys().len(), 3); + assert_eq!(settings.keys().len(), 4); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["facetedAttributes"], json!({})); + assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximmity", "attributes", "wordsPosition", "exactness"])); } #[actix_rt::test] From 7d28f8cff0fda748663e53dc27373496ee064eba Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 10:51:52 +0100 Subject: [PATCH 136/522] implement get single udpate --- Cargo.lock | 1 + Cargo.toml | 2 +- src/data/updates.rs | 6 ++---- src/index_controller/mod.rs | 9 +++++++-- src/index_controller/update_actor.rs | 26 ++++++++++++++++++++++++++ src/routes/index.rs | 2 +- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f1134379..cbae20c98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1870,6 +1870,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" +source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" dependencies = [ "anyhow", "bstr", diff --git a/Cargo.toml b/Cargo.toml index aaf571ef3..9bc5e4d1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { path = "../milli/milli" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" diff --git a/src/data/updates.rs b/src/data/updates.rs index 3e3837861..dd74bc47f 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -59,10 +59,8 @@ impl Data { //Ok(()) } - #[inline] - pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { - todo!() - //self.index_controller.update_status(index, uid) + pub async fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { + self.index_controller.update_status(index.as_ref().to_string(), uid).await } pub async fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 8eb1684a2..4606526c4 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -150,8 +150,13 @@ impl IndexController { todo!() } - fn update_status(&self, index: String, id: u64) -> anyhow::Result> { - todo!() + pub async fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + let uuid = self.uuid_resolver + .resolve(index) + .await? + .context("index not found")?; + let result = self.update_handle.update_status(uuid, id).await?; + Ok(result) } pub async fn all_update_status(&self, index: String) -> anyhow::Result> { diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index b236df2a6..83f29c380 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -38,6 +38,11 @@ enum UpdateMsg { uuid: Uuid, ret: oneshot::Sender>>, }, + GetUpdate { + uuid: Uuid, + ret: oneshot::Sender>>, + id: u64, + } } struct UpdateActor { @@ -81,6 +86,9 @@ where Some(ListUpdates { uuid, ret }) => { let _ = ret.send(self.handle_list_updates(uuid).await); } , + Some(GetUpdate { uuid, ret, id }) => { + let _ = ret.send(self.handle_get_update(uuid, id).await); + } None => {} } } @@ -154,6 +162,17 @@ where }).await .map_err(|e| UpdateError::Error(Box::new(e)))? } + + + async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result> { + let store = self.store + .get(&uuid) + .await? + .ok_or(UpdateError::UnexistingIndex(uuid))?; + let result = store.meta(id) + .map_err(|e| UpdateError::Error(Box::new(e)))?; + Ok(result) + } } #[derive(Clone)] @@ -199,6 +218,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn update_status(&self, uuid: Uuid, id: u64) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::GetUpdate { uuid, id, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { diff --git a/src/routes/index.rs b/src/routes/index.rs index a8618d9dd..c9200f085 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -133,7 +133,7 @@ async fn get_update_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_update_status(&path.index_uid, path.update_id); + let result = data.get_update_status(&path.index_uid, path.update_id).await; match result { Ok(Some(meta)) => { let json = serde_json::to_string(&meta).unwrap(); From 86211b1ddd0c928bea2222d9341c25673102e1d1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 10:53:11 +0100 Subject: [PATCH 137/522] import routes modules in main --- src/main.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index c5e2d11e6..9e69c4091 100644 --- a/src/main.rs +++ b/src/main.rs @@ -83,18 +83,20 @@ async fn main() -> Result<(), MainError> { } async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box> { + use meilisearch::routes::*; + let http_server = HttpServer::new(move || { let app = App::new() .configure(|c| configure_data(c, &data)) - .configure(meilisearch::routes::document::services) - .configure(meilisearch::routes::index::services) - .configure(meilisearch::routes::search::services) - .configure(meilisearch::routes::settings::services) - .configure(meilisearch::routes::stop_words::services) - .configure(meilisearch::routes::synonym::services) - .configure(meilisearch::routes::health::services) - .configure(meilisearch::routes::stats::services) - .configure(meilisearch::routes::key::services); + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(stop_words::services) + .configure(synonym::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services); //.configure(routes::dump::services); let app = if enable_frontend { app From d9254c43554755f8cc8e153f1d5b52dd029b2d58 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 12:57:56 +0100 Subject: [PATCH 138/522] implement index delete --- src/data/updates.rs | 8 +-- src/index_controller/index_actor.rs | 80 +++++++++++++++++++++++---- src/index_controller/mod.rs | 32 +++++++++-- src/index_controller/update_actor.rs | 51 +++++++++++++++-- src/index_controller/uuid_resolver.rs | 30 ++++++++-- 5 files changed, 171 insertions(+), 30 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index dd74bc47f..e9d1b51b7 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -51,12 +51,10 @@ impl Data { pub async fn delete_index( &self, - _index: impl AsRef + Send + Sync + 'static, + index: impl AsRef + Send + Sync + 'static, ) -> anyhow::Result<()> { - todo!() - //let index_controller = self.index_controller.clone(); - //tokio::task::spawn_blocking(move || { index_controller.delete_index(index) }).await??; - //Ok(()) + self.index_controller.delete_index(index.as_ref().to_owned()).await?; + Ok(()) } pub async fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 0e74d6665..c77793a5b 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -1,19 +1,20 @@ use std::collections::{hash_map::Entry, HashMap}; -use std::fs::{create_dir_all, File}; +use std::fs::{create_dir_all, File, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; use chrono::Utc; +use futures::pin_mut; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; +use std::future::Future; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; -use std::future::Future; -use futures::pin_mut; +use super::get_arc_ownership_blocking; use super::update_handler::UpdateHandler; use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; @@ -59,7 +60,11 @@ enum IndexMsg { attributes_to_retrieve: Option>, doc_id: String, ret: oneshot::Sender>, - } + }, + Delete { + uuid: Uuid, + ret: oneshot::Sender>, + }, } struct IndexActor { @@ -82,13 +87,14 @@ trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get_or_create(&self, uuid: Uuid) -> Result; async fn get(&self, uuid: Uuid) -> Result>; + async fn delete(&self, uuid: &Uuid) -> Result>; } impl IndexActor { fn new( read_receiver: mpsc::Receiver, write_receiver: mpsc::Receiver, - store: S + store: S, ) -> Self { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).unwrap(); @@ -149,7 +155,6 @@ impl IndexActor { //futures.push(fut2); //futures.for_each(f) tokio::join!(fut1, fut2); - } async fn handle_message(&self, msg: IndexMsg) { @@ -173,7 +178,18 @@ impl IndexActor { self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) .await } - Document { uuid, attributes_to_retrieve, doc_id, ret } => self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret).await, + Document { + uuid, + attributes_to_retrieve, + doc_id, + ret, + } => { + self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret) + .await + } + Delete { uuid, ret } => { + let _ = ret.send(self.handle_delete(uuid).await); + }, } } @@ -236,10 +252,12 @@ impl IndexActor { ) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - let result = index.retrieve_documents(offset, limit, attributes_to_retrieve) + let result = index + .retrieve_documents(offset, limit, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)); let _ = ret.send(result); - }).await; + }) + .await; } async fn handle_fetch_document( @@ -251,10 +269,29 @@ impl IndexActor { ) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - let result = index.retrieve_document(doc_id, attributes_to_retrieve) + let result = index + .retrieve_document(doc_id, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)); let _ = ret.send(result); - }).await; + }) + .await; + } + + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + let index = self.store.delete(&uuid).await?; + + if let Some(index) = index { + tokio::task::spawn(async move { + let index = index.0; + let store = get_arc_ownership_blocking(index).await; + tokio::task::spawn_blocking(move || { + store.prepare_for_closing().wait(); + info!("Index {} was closed.", uuid); + }); + }); + } + + Ok(()) } } @@ -272,7 +309,10 @@ impl IndexActorHandle { let store = MapIndexStore::new(path); let actor = IndexActor::new(read_receiver, write_receiver, store); tokio::task::spawn(actor.run()); - Self { read_sender, write_sender } + Self { + read_sender, + write_sender, + } } pub async fn create_index( @@ -346,6 +386,13 @@ impl IndexActorHandle { let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn delete(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Delete { uuid, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { @@ -408,6 +455,15 @@ impl IndexStore for MapIndexStore { async fn get(&self, uuid: Uuid) -> Result> { Ok(self.index_store.read().await.get(&uuid).cloned()) } + + async fn delete(&self, uuid: &Uuid) -> Result> { + let index = self.index_store.write().await.remove(&uuid); + if index.is_some() { + let db_path = self.root.join(format!("index-{}", uuid)); + remove_dir_all(db_path).unwrap(); + } + Ok(index) + } } impl MapIndexStore { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 4606526c4..19fe62f4d 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,11 +1,13 @@ -mod updates; mod index_actor; mod update_actor; -mod uuid_resolver; -mod update_store; mod update_handler; +mod update_store; +mod updates; +mod uuid_resolver; use std::path::Path; +use std::sync::Arc; +use std::time::Duration; use actix_web::web::{Bytes, Payload}; use anyhow::Context; @@ -14,6 +16,7 @@ use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; use tokio::sync::{mpsc, oneshot}; +use tokio::time::sleep; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; @@ -146,8 +149,14 @@ impl IndexController { Ok(index_meta) } - fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { - todo!() + pub async fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { + let uuid = self.uuid_resolver + .delete(index_uid) + .await? + .context("index not found")?; + self.update_handle.delete(uuid.clone()).await?; + self.index_handle.delete(uuid).await?; + Ok(()) } pub async fn update_status(&self, index: String, id: u64) -> anyhow::Result> { @@ -219,3 +228,16 @@ impl IndexController { Ok(result) } } + +pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { + loop { + match Arc::try_unwrap(item) { + Ok(item) => return item, + Err(item_arc) => { + item = item_arc; + sleep(Duration::from_millis(100)).await; + continue; + } + } + } +} diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index 83f29c380..370912dcf 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -1,19 +1,20 @@ use std::collections::{hash_map::Entry, HashMap}; -use std::fs::create_dir_all; +use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use super::index_actor::IndexActorHandle; +use itertools::Itertools; use log::info; +use super::index_actor::IndexActorHandle; use thiserror::Error; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; -use itertools::Itertools; use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; +use super::get_arc_ownership_blocking; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; @@ -42,7 +43,11 @@ enum UpdateMsg { uuid: Uuid, ret: oneshot::Sender>>, id: u64, - } + }, + Delete { + uuid: Uuid, + ret: oneshot::Sender>, + }, } struct UpdateActor { @@ -54,6 +59,7 @@ struct UpdateActor { #[async_trait::async_trait] trait UpdateStoreStore { async fn get_or_create(&self, uuid: Uuid) -> Result>; + async fn delete(&self, uuid: &Uuid) -> Result>>; async fn get(&self, uuid: &Uuid) -> Result>>; } @@ -89,6 +95,9 @@ where Some(GetUpdate { uuid, ret, id }) => { let _ = ret.send(self.handle_get_update(uuid, id).await); } + Some(Delete { uuid, ret }) => { + let _ = ret.send(self.handle_delete(uuid).await); + } None => {} } } @@ -173,6 +182,24 @@ where .map_err(|e| UpdateError::Error(Box::new(e)))?; Ok(result) } + + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + let store = self.store + .delete(&uuid) + .await?; + + if let Some(store) = store { + tokio::task::spawn(async move { + let store = get_arc_ownership_blocking(store).await; + tokio::task::spawn_blocking(move || { + store.prepare_for_closing().wait(); + info!("Update store {} was closed.", uuid); + }); + }); + } + + Ok(()) + } } #[derive(Clone)] @@ -225,6 +252,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn delete(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Delete { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { @@ -269,4 +303,13 @@ impl UpdateStoreStore for MapUpdateStoreStore { async fn get(&self, uuid: &Uuid) -> Result>> { Ok(self.db.read().await.get(uuid).cloned()) } + + async fn delete(&self, uuid: &Uuid) -> Result>> { + let store = self.db.write().await.remove(&uuid); + if store.is_some() { + let path = self.path.clone().join(format!("updates-{}", uuid)); + remove_dir_all(path).unwrap(); + } + Ok(store) + } } diff --git a/src/index_controller/uuid_resolver.rs b/src/index_controller/uuid_resolver.rs index 3143ae8fc..50740f30f 100644 --- a/src/index_controller/uuid_resolver.rs +++ b/src/index_controller/uuid_resolver.rs @@ -22,6 +22,10 @@ enum UuidResolveMsg { name: String, ret: oneshot::Sender>, }, + Delete { + name: String, + ret: oneshot::Sender>>, + }, } struct UuidResolverActor { @@ -45,6 +49,7 @@ impl UuidResolverActor { Some(Create { name, ret }) => self.handle_create(name, ret).await, Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, Some(Resolve { name, ret }) => self.handle_resolve(name, ret).await, + Some(Delete { name, ret }) => self.handle_delete(name, ret).await, // all senders have been dropped, need to quit. None => break, } @@ -64,7 +69,12 @@ impl UuidResolverActor { } async fn handle_resolve(&self, name: String, ret: oneshot::Sender>>) { - let result = self.store.get_uuid(name).await; + let result = self.store.get_uuid(&name).await; + let _ = ret.send(result); + } + + async fn handle_delete(&self, name: String, ret: oneshot::Sender>>) { + let result = self.store.delete(&name).await; let _ = ret.send(result); } } @@ -103,6 +113,13 @@ impl UuidResolverHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("Uuid resolver actor has been killed")?) } + + pub async fn delete(&self, name: String) -> anyhow::Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Delete { name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } } #[derive(Clone, Debug, Error)] @@ -116,7 +133,8 @@ trait UuidStore { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return // the uuid otherwise. async fn create_uuid(&self, name: String, err: bool) -> Result; - async fn get_uuid(&self, name: String) -> Result>; + async fn get_uuid(&self, name: &str) -> Result>; + async fn delete(&self, name: &str) -> Result>; } struct MapUuidStore(Arc>>); @@ -140,7 +158,11 @@ impl UuidStore for MapUuidStore { } } - async fn get_uuid(&self, name: String) -> Result> { - Ok(self.0.read().await.get(&name).cloned()) + async fn get_uuid(&self, name: &str) -> Result> { + Ok(self.0.read().await.get(name).cloned()) + } + + async fn delete(&self, name: &str) -> Result> { + Ok(self.0.write().await.remove(name)) } } From 281a4459988cf9e25464be2ff72818fdee8ddfcc Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 20:12:20 +0100 Subject: [PATCH 139/522] implement list indexes --- src/data/mod.rs | 4 +- src/index_controller/index_actor.rs | 62 ++++++++++++++++++++------- src/index_controller/mod.rs | 32 +++++++++----- src/index_controller/uuid_resolver.rs | 24 +++++++++++ src/routes/index.rs | 2 +- 5 files changed, 95 insertions(+), 29 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 1dec00766..cdb2127a8 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -82,8 +82,8 @@ impl Data { self.index_controller.settings(index_uid.as_ref().to_string()).await } - pub fn list_indexes(&self) -> anyhow::Result> { - self.index_controller.list_indexes() + pub async fn list_indexes(&self) -> anyhow::Result> { + self.index_controller.list_indexes().await } pub fn index(&self, name: impl AsRef) -> anyhow::Result> { diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index c77793a5b..360f3a3c0 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -1,18 +1,19 @@ use std::collections::{hash_map::Entry, HashMap}; -use std::fs::{create_dir_all, File, remove_dir_all}; +use std::fs::{create_dir_all, remove_dir_all, File}; +use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; -use chrono::Utc; +use chrono::{Utc, DateTime}; use futures::pin_mut; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; -use std::future::Future; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; +use serde::{Serialize, Deserialize}; use super::get_arc_ownership_blocking; use super::update_handler::UpdateHandler; @@ -20,7 +21,7 @@ use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; use crate::index_controller::{ updates::{Failed, Processed, Processing}, - IndexMetadata, UpdateMeta, + UpdateMeta, }; use crate::option::IndexerOpts; @@ -28,11 +29,20 @@ pub type Result = std::result::Result; type AsyncMap = Arc>>; type UpdateResult = std::result::Result, Failed>; +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IndexMeta{ + uuid: Uuid, + created_at: DateTime, + updated_at: DateTime, + primary_key: Option, +} + enum IndexMsg { CreateIndex { uuid: Uuid, primary_key: Option, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, Update { meta: Processing, @@ -65,6 +75,10 @@ enum IndexMsg { uuid: Uuid, ret: oneshot::Sender>, }, + GetMeta { + uuid: Uuid, + ret: oneshot::Sender>>, + }, } struct IndexActor { @@ -84,10 +98,11 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get_or_create(&self, uuid: Uuid) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: &Uuid) -> Result>; + async fn get_meta(&self, uuid: &Uuid) -> Result>; } impl IndexActor { @@ -150,10 +165,6 @@ impl IndexActor { let fut1: Box + Unpin + Send> = Box::new(fut1); let fut2: Box + Unpin + Send> = Box::new(fut2); - //let futures = futures::stream::futures_unordered::FuturesUnordered::new(); - //futures.push(fut1); - //futures.push(fut2); - //futures.for_each(f) tokio::join!(fut1, fut2); } @@ -189,7 +200,10 @@ impl IndexActor { } Delete { uuid, ret } => { let _ = ret.send(self.handle_delete(uuid).await); - }, + } + GetMeta { uuid, ret } => { + let _ = ret.send(self.handle_get_meta(uuid).await); + } } } @@ -210,7 +224,7 @@ impl IndexActor { &self, uuid: Uuid, primary_key: Option, - ret: oneshot::Sender>, + ret: oneshot::Sender>, ) { let result = self.store.create_index(uuid, primary_key).await; let _ = ret.send(result); @@ -293,6 +307,11 @@ impl IndexActor { Ok(()) } + + async fn handle_get_meta(&self, uuid: Uuid) -> Result> { + let result = self.store.get_meta(&uuid).await?; + Ok(result) + } } #[derive(Clone)] @@ -319,7 +338,7 @@ impl IndexActorHandle { &self, uuid: Uuid, primary_key: Option, - ) -> Result { + ) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { ret, @@ -393,20 +412,27 @@ impl IndexActorHandle { let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn get_index_meta(&self, uuid: Uuid) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::GetMeta { uuid, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { root: PathBuf, - meta_store: AsyncMap, + meta_store: AsyncMap, index_store: AsyncMap, } #[async_trait::async_trait] impl IndexStore for MapIndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let meta = match self.meta_store.write().await.entry(uuid.clone()) { Entry::Vacant(entry) => { - let meta = IndexMetadata { + let meta = IndexMeta{ uuid, created_at: Utc::now(), updated_at: Utc::now(), @@ -464,6 +490,10 @@ impl IndexStore for MapIndexStore { } Ok(index) } + + async fn get_meta(&self, uuid: &Uuid) -> Result> { + Ok(self.meta_store.read().await.get(uuid).cloned()) + } } impl MapIndexStore { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 19fe62f4d..09a979cbd 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -11,7 +11,6 @@ use std::time::Duration; use actix_web::web::{Bytes, Payload}; use anyhow::Context; -use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; @@ -28,10 +27,9 @@ pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { - uuid: Uuid, - created_at: DateTime, - updated_at: DateTime, - primary_key: Option, + name: String, + #[serde(flatten)] + meta: index_actor::IndexMeta, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -144,9 +142,12 @@ impl IndexController { pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { let IndexSettings { name, primary_key } = index_settings; - let uuid = self.uuid_resolver.create(name.unwrap()).await?; - let index_meta = self.index_handle.create_index(uuid, primary_key).await?; - Ok(index_meta) + let name = name.unwrap(); + let uuid = self.uuid_resolver.create(name.clone()).await?; + let meta = self.index_handle.create_index(uuid, primary_key).await?; + let meta = IndexMetadata { name, meta }; + + Ok(meta) } pub async fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { @@ -176,8 +177,19 @@ impl IndexController { Ok(result) } - pub fn list_indexes(&self) -> anyhow::Result> { - todo!() + pub async fn list_indexes(&self) -> anyhow::Result> { + let uuids = self.uuid_resolver.list().await?; + + let mut ret = Vec::new(); + + for (name, uuid) in uuids { + if let Some(meta) = self.index_handle.get_index_meta(uuid).await? { + let meta = IndexMetadata { name, meta }; + ret.push(meta); + } + } + + Ok(ret) } pub async fn settings(&self, index: String) -> anyhow::Result { diff --git a/src/index_controller/uuid_resolver.rs b/src/index_controller/uuid_resolver.rs index 50740f30f..e2539fdb2 100644 --- a/src/index_controller/uuid_resolver.rs +++ b/src/index_controller/uuid_resolver.rs @@ -26,6 +26,9 @@ enum UuidResolveMsg { name: String, ret: oneshot::Sender>>, }, + List { + ret: oneshot::Sender>>, + }, } struct UuidResolverActor { @@ -50,6 +53,9 @@ impl UuidResolverActor { Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, Some(Resolve { name, ret }) => self.handle_resolve(name, ret).await, Some(Delete { name, ret }) => self.handle_delete(name, ret).await, + Some(List { ret }) => { + let _ = ret.send(self.handle_list().await); + } // all senders have been dropped, need to quit. None => break, } @@ -77,6 +83,11 @@ impl UuidResolverActor { let result = self.store.delete(&name).await; let _ = ret.send(result); } + + async fn handle_list(&self) -> Result> { + let result = self.store.list().await?; + Ok(result) + } } #[derive(Clone)] @@ -120,6 +131,13 @@ impl UuidResolverHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("Uuid resolver actor has been killed")?) } + + pub async fn list(&self) -> anyhow::Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::List { ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } } #[derive(Clone, Debug, Error)] @@ -135,6 +153,7 @@ trait UuidStore { async fn create_uuid(&self, name: String, err: bool) -> Result; async fn get_uuid(&self, name: &str) -> Result>; async fn delete(&self, name: &str) -> Result>; + async fn list(&self) -> Result>; } struct MapUuidStore(Arc>>); @@ -165,4 +184,9 @@ impl UuidStore for MapUuidStore { async fn delete(&self, name: &str) -> Result> { Ok(self.0.write().await.remove(name)) } + + async fn list(&self) -> Result> { + let list = self.0.read().await.iter().map(|(name, uuid)| (name.to_owned(), uuid.clone())).collect(); + Ok(list) + } } diff --git a/src/routes/index.rs b/src/routes/index.rs index c9200f085..752493a3b 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -21,7 +21,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { #[get("/indexes", wrap = "Authentication::Private")] async fn list_indexes(data: web::Data) -> Result { - match data.list_indexes() { + match data.list_indexes().await { Ok(indexes) => { let json = serde_json::to_string(&indexes).unwrap(); Ok(HttpResponse::Ok().body(&json)) From ced32afd9fb102385bf4089e69db8c1382ed1574 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 20:17:58 +0100 Subject: [PATCH 140/522] implement get single index --- src/data/mod.rs | 8 ++------ src/index_controller/mod.rs | 12 ++++++++++++ src/routes/index.rs | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index cdb2127a8..0f7722657 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -86,12 +86,8 @@ impl Data { self.index_controller.list_indexes().await } - pub fn index(&self, name: impl AsRef) -> anyhow::Result> { - todo!() - //Ok(self - //.list_indexes()? - //.into_iter() - //.find(|i| i.uid == name.as_ref())) + pub async fn index(&self, name: impl AsRef) -> anyhow::Result> { + self.index_controller.get_index(name.as_ref().to_string()).await } pub async fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 09a979cbd..bcf663ec8 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -239,6 +239,18 @@ impl IndexController { let result = self.index_handle.search(uuid, query).await?; Ok(result) } + + pub async fn get_index(&self, name: String) -> anyhow::Result> { + let uuid = self.uuid_resolver.resolve(name.clone()).await?; + if let Some(uuid) = uuid { + let result = self.index_handle + .get_index_meta(uuid) + .await? + .map(|meta| IndexMetadata { name, meta }); + return Ok(result) + } + Ok(None) + } } pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { diff --git a/src/routes/index.rs b/src/routes/index.rs index 752493a3b..818c87303 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -37,7 +37,7 @@ async fn get_index( data: web::Data, path: web::Path, ) -> Result { - match data.index(&path.index_uid)? { + match data.index(&path.index_uid).await? { Some(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) From ac4d795eff4a625080f3da9dd48265ebc52c2e51 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 10:20:53 +0100 Subject: [PATCH 141/522] update created at when updating index --- src/index_controller/index_actor.rs | 42 +++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 360f3a3c0..2556e224a 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -94,11 +94,16 @@ pub enum IndexError { Error(#[from] anyhow::Error), #[error("index already exists")] IndexAlreadyExists, + #[error("Index doesn't exists")] + UnexistingIndex, } #[async_trait::async_trait] trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn update_index(&self, uuid: Uuid, f: F) -> Result + where F: FnOnce(Index) -> Result + Send + Sync + 'static, + R: Sync + Send + 'static; async fn get_or_create(&self, uuid: Uuid) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: &Uuid) -> Result>; @@ -238,13 +243,16 @@ impl IndexActor { ) { info!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); - let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); - tokio::task::spawn_blocking(move || { - let result = update_handler.handle_update(meta, data, index); - let _ = ret.send(result); - }) - .await; + let handle = self.store.update_index(uuid, |index| { + let handle = tokio::task::spawn_blocking(move || { + let result = update_handler.handle_update(meta, data, index); + let _ = ret.send(result); + }); + Ok(handle) + }); + + handle.await; } async fn handle_settings(&self, uuid: Uuid, ret: oneshot::Sender>) { @@ -432,10 +440,11 @@ impl IndexStore for MapIndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let meta = match self.meta_store.write().await.entry(uuid.clone()) { Entry::Vacant(entry) => { + let now = Utc::now(); let meta = IndexMeta{ uuid, - created_at: Utc::now(), - updated_at: Utc::now(), + created_at: now.clone(), + updated_at: now, primary_key, }; entry.insert(meta).clone() @@ -494,6 +503,23 @@ impl IndexStore for MapIndexStore { async fn get_meta(&self, uuid: &Uuid) -> Result> { Ok(self.meta_store.read().await.get(uuid).cloned()) } + + async fn update_index(&self, uuid: Uuid, f: F) -> Result + where F: FnOnce(Index) -> Result + Send + Sync + 'static, + R: Sync + Send + 'static, + { + let index = self.get_or_create(uuid.clone()).await?; + let mut meta = self.get_meta(&uuid).await? + .ok_or(IndexError::UnexistingIndex)?; + match f(index) { + Ok(r) => { + meta.updated_at = Utc::now(); + self.meta_store.write().await.insert(uuid, meta); + Ok(r) + } + Err(e) => Err(e) + } + } } impl MapIndexStore { From 2f93cce7aaf371adff969264291a313f08e47417 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 10:48:34 +0100 Subject: [PATCH 142/522] auto index creation --- src/index_controller/index_actor.rs | 49 ++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 2556e224a..c85224bb4 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -104,7 +104,7 @@ trait IndexStore { async fn update_index(&self, uuid: Uuid, f: F) -> Result where F: FnOnce(Index) -> Result + Send + Sync + 'static, R: Sync + Send + 'static; - async fn get_or_create(&self, uuid: Uuid) -> Result; + async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: &Uuid) -> Result>; async fn get_meta(&self, uuid: &Uuid) -> Result>; @@ -473,14 +473,49 @@ impl IndexStore for MapIndexStore { Ok(meta) } - async fn get_or_create(&self, uuid: Uuid) -> Result { + async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result { match self.index_store.write().await.entry(uuid.clone()) { - Entry::Vacant(entry) => match self.meta_store.write().await.entry(uuid.clone()) { - Entry::Vacant(_) => { - todo!() + Entry::Vacant(index_entry) => match self.meta_store.write().await.entry(uuid.clone()) { + Entry::Vacant(meta_entry) => { + let now = Utc::now(); + let meta = IndexMeta{ + uuid, + created_at: now.clone(), + updated_at: now, + primary_key, + }; + let meta = meta_entry.insert(meta); + let db_path = self.root.join(format!("index-{}", meta.uuid)); + + let index: Result = tokio::task::spawn_blocking(move || { + create_dir_all(&db_path).expect("can't create db"); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; + let index = Index(Arc::new(index)); + Ok(index) + }) + .await + .expect("thread died"); + + Ok(index_entry.insert(index?).clone()) } Entry::Occupied(entry) => { - todo!() + let meta = entry.get(); + let db_path = self.root.join(format!("index-{}", meta.uuid)); + + let index: Result = tokio::task::spawn_blocking(move || { + create_dir_all(&db_path).expect("can't create db"); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; + let index = Index(Arc::new(index)); + Ok(index) + }) + .await + .expect("thread died"); + + Ok(index_entry.insert(index?).clone()) } }, Entry::Occupied(entry) => Ok(entry.get().clone()), @@ -508,7 +543,7 @@ impl IndexStore for MapIndexStore { where F: FnOnce(Index) -> Result + Send + Sync + 'static, R: Sync + Send + 'static, { - let index = self.get_or_create(uuid.clone()).await?; + let index = self.get_or_create(uuid.clone(), None).await?; let mut meta = self.get_meta(&uuid).await? .ok_or(IndexError::UnexistingIndex)?; match f(index) { From 944a5bb36e1cecc34af95c2d951d5c4bc181452a Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 13:28:31 +0100 Subject: [PATCH 143/522] update milli --- Cargo.lock | 3 ++- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/data/mod.rs | 2 +- meilisearch-http/src/data/search.rs | 12 ++++++------ meilisearch-http/tests/settings/get_settings.rs | 3 +-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17575cd48..c54acfa02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1725,7 +1725,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" +source = "git+https://github.com/meilisearch/milli.git?rev=f190d5f#f190d5f496cc39517b6a81300c6dee9b6dba7a38" dependencies = [ "anyhow", "bstr", @@ -1756,6 +1756,7 @@ dependencies = [ "roaring", "serde", "serde_json", + "slice-group-by", "smallstr", "smallvec", "tempfile", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index b760a3d27..8ee612d3f 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -38,7 +38,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "f190d5f" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 15db0e4ae..2904bb67f 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -109,7 +109,7 @@ impl Data { let criteria = index .criteria(&txn)? .into_iter() - .map(|v| format!("{:?}", v)) + .map(|v| format!("{}", v)) .collect(); Ok(Settings { diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index c76e21638..a721a8200 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -6,7 +6,7 @@ use anyhow::{bail, Context}; use either::Either; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{obkv_to_json, FacetCondition, Index, facet::FacetValue}; +use milli::{FacetCondition, Index, MatchingWords, facet::FacetValue, obkv_to_json}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -60,7 +60,7 @@ impl SearchQuery { let milli::SearchResult { documents_ids, - found_words, + matching_words, candidates, .. } = search.execute()?; @@ -92,7 +92,7 @@ impl SearchQuery { for (_id, obkv) in index.documents(&rtxn, documents_ids)? { let mut object = obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv)?; if let Some(ref attributes_to_highlight) = self.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } documents.push(object); } @@ -145,7 +145,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { Self { analyzer } } - fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + fn highlight_value(&self, value: Value, words_to_highlight: &MatchingWords) -> Value { match value { Value::Null => Value::Null, Value::Bool(boolean) => Value::Bool(boolean), @@ -155,7 +155,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { let analyzed = self.analyzer.analyze(&old_string); for (word, token) in analyzed.reconstruct() { if token.is_word() { - let to_highlight = words_to_highlight.contains(token.text()); + let to_highlight = words_to_highlight.matches(token.text()); if to_highlight { string.push_str("") } @@ -187,7 +187,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { fn highlight_record( &self, object: &mut Map, - words_to_highlight: &HashSet, + words_to_highlight: &MatchingWords, attributes_to_highlight: &HashSet, ) { // TODO do we need to create a string for element that are not and needs to be highlight? diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 6c57863ac..25d46b961 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -10,7 +10,6 @@ async fn get_settings_unexisting_index() { // test broken, should be fixed with milli#101 #[actix_rt::test] -#[ignore] async fn get_settings() { let server = Server::new().await; let index = server.index("test"); @@ -22,7 +21,7 @@ async fn get_settings() { assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["facetedAttributes"], json!({})); - assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximmity", "attributes", "wordsPosition", "exactness"])); + assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximity", "attribute", "wordsPosition", "exactness"])); } #[actix_rt::test] From e9b90d5380be9a89a31637898bd15d5fb33f102d Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 13:51:33 +0100 Subject: [PATCH 144/522] fixes from review --- meilisearch-http/src/data/mod.rs | 2 +- meilisearch-http/src/data/search.rs | 1 - meilisearch-http/tests/settings/get_settings.rs | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 2904bb67f..05a29381a 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -109,7 +109,7 @@ impl Data { let criteria = index .criteria(&txn)? .into_iter() - .map(|v| format!("{}", v)) + .map(|v| v.to_string()) .collect(); Ok(Settings { diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index a721a8200..7692417ed 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -62,7 +62,6 @@ impl SearchQuery { documents_ids, matching_words, candidates, - .. } = search.execute()?; let mut documents = Vec::new(); diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 25d46b961..d1ce6ea5d 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -8,7 +8,6 @@ async fn get_settings_unexisting_index() { assert_eq!(code, 400) } -// test broken, should be fixed with milli#101 #[actix_rt::test] async fn get_settings() { let server = Server::new().await; From 06403a57081a63805baee44ade6232e2500df5e8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 15:53:16 +0100 Subject: [PATCH 145/522] clean index actor unwraps --- src/data/mod.rs | 4 +- src/index_controller/index_actor.rs | 188 +++++++++++++++------------ src/index_controller/mod.rs | 6 +- src/index_controller/update_store.rs | 12 +- 4 files changed, 113 insertions(+), 97 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 0f7722657..b011c18b3 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -59,10 +59,8 @@ impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - //let indexer_opts = options.indexer_options.clone(); - create_dir_all(&path)?; - let index_controller = IndexController::new(&path); + let index_controller = IndexController::new(&path)?; let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index c85224bb4..1ddc041a1 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -5,15 +5,15 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; -use chrono::{Utc, DateTime}; +use chrono::{DateTime, Utc}; use futures::pin_mut; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; +use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; -use serde::{Serialize, Deserialize}; use super::get_arc_ownership_blocking; use super::update_handler::UpdateHandler; @@ -31,7 +31,7 @@ type UpdateResult = std::result::Result, Failed, updated_at: DateTime, @@ -47,7 +47,7 @@ enum IndexMsg { Update { meta: Processing, data: std::fs::File, - ret: oneshot::Sender, + ret: oneshot::Sender>, }, Search { uuid: Uuid, @@ -102,8 +102,9 @@ pub enum IndexError { trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn update_index(&self, uuid: Uuid, f: F) -> Result - where F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static; + where + F: FnOnce(Index) -> Result + Send + Sync + 'static, + R: Sync + Send + 'static; async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: &Uuid) -> Result>; @@ -115,18 +116,19 @@ impl IndexActor { read_receiver: mpsc::Receiver, write_receiver: mpsc::Receiver, store: S, - ) -> Self { + ) -> Result { let options = IndexerOpts::default(); - let update_handler = UpdateHandler::new(&options).unwrap(); + let update_handler = UpdateHandler::new(&options) + .map_err(|e| IndexError::Error(e.into()))?; let update_handler = Arc::new(update_handler); let read_receiver = Some(read_receiver); let write_receiver = Some(write_receiver); - Self { + Ok(Self { read_receiver, write_receiver, store, update_handler, - } + }) } /// `run` poll the write_receiver and read_receiver concurrently, but while messages send @@ -180,10 +182,18 @@ impl IndexActor { uuid, primary_key, ret, - } => self.handle_create_index(uuid, primary_key, ret).await, - Update { ret, meta, data } => self.handle_update(meta, data, ret).await, - Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, - Settings { ret, uuid } => self.handle_settings(uuid, ret).await, + } => { + let _ = ret.send(self.handle_create_index(uuid, primary_key).await); + } + Update { ret, meta, data } => { + let _ = ret.send(self.handle_update(meta, data).await); + } + Search { ret, query, uuid } => { + let _ = ret.send(self.handle_search(uuid, query).await); + } + Settings { ret, uuid } => { + let _ = ret.send(self.handle_settings(uuid).await); + } Documents { ret, uuid, @@ -191,8 +201,10 @@ impl IndexActor { offset, limit, } => { - self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) - .await + let _ = ret.send( + self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve) + .await, + ); } Document { uuid, @@ -200,8 +212,10 @@ impl IndexActor { doc_id, ret, } => { - self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret) - .await + let _ = ret.send( + self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve) + .await, + ); } Delete { uuid, ret } => { let _ = ret.send(self.handle_delete(uuid).await); @@ -212,56 +226,51 @@ impl IndexActor { } } - async fn handle_search( - &self, - uuid: Uuid, - query: SearchQuery, - ret: oneshot::Sender>, - ) { - let index = self.store.get(uuid).await.unwrap().unwrap(); - tokio::task::spawn_blocking(move || { - let result = index.perform_search(query); - ret.send(result) - }); + async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> anyhow::Result { + let index = self.store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + tokio::task::spawn_blocking(move || index.perform_search(query)).await? } async fn handle_create_index( &self, uuid: Uuid, primary_key: Option, - ret: oneshot::Sender>, - ) { - let result = self.store.create_index(uuid, primary_key).await; - let _ = ret.send(result); + ) -> Result { + self.store.create_index(uuid, primary_key).await } async fn handle_update( &self, meta: Processing, data: File, - ret: oneshot::Sender, - ) { + ) -> Result { info!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let update_handler = self.update_handler.clone(); - let handle = self.store.update_index(uuid, |index| { - let handle = tokio::task::spawn_blocking(move || { - let result = update_handler.handle_update(meta, data, index); - let _ = ret.send(result); - }); - Ok(handle) - }); + let handle = self + .store + .update_index(uuid, |index| { + let handle = tokio::task::spawn_blocking(move || { + update_handler.handle_update(meta, data, index) + }); + Ok(handle) + }) + .await?; - handle.await; + handle.await.map_err(|e| IndexError::Error(e.into())) } - async fn handle_settings(&self, uuid: Uuid, ret: oneshot::Sender>) { - let index = self.store.get(uuid).await.unwrap().unwrap(); - tokio::task::spawn_blocking(move || { - let result = index.settings().map_err(|e| IndexError::Error(e)); - let _ = ret.send(result); - }) - .await; + async fn handle_settings(&self, uuid: Uuid) -> Result { + let index = self.store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + tokio::task::spawn_blocking(move || index.settings().map_err(|e| IndexError::Error(e))) + .await + .map_err(|e| IndexError::Error(e.into()))? } async fn handle_fetch_documents( @@ -270,16 +279,17 @@ impl IndexActor { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ret: oneshot::Sender>>, - ) { - let index = self.store.get(uuid).await.unwrap().unwrap(); + ) -> Result> { + let index = self.store.get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; tokio::task::spawn_blocking(move || { - let result = index + index .retrieve_documents(offset, limit, attributes_to_retrieve) - .map_err(|e| IndexError::Error(e)); - let _ = ret.send(result); + .map_err(|e| IndexError::Error(e)) }) - .await; + .await + .map_err(|e| IndexError::Error(e.into()))? } async fn handle_fetch_document( @@ -287,16 +297,19 @@ impl IndexActor { uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ret: oneshot::Sender>, - ) { - let index = self.store.get(uuid).await.unwrap().unwrap(); + ) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; tokio::task::spawn_blocking(move || { - let result = index + index .retrieve_document(doc_id, attributes_to_retrieve) - .map_err(|e| IndexError::Error(e)); - let _ = ret.send(result); + .map_err(|e| IndexError::Error(e)) }) - .await; + .await + .map_err(|e| IndexError::Error(e.into()))? } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { @@ -329,24 +342,20 @@ pub struct IndexActorHandle { } impl IndexActorHandle { - pub fn new(path: impl AsRef) -> Self { + pub fn new(path: impl AsRef) -> anyhow::Result { let (read_sender, read_receiver) = mpsc::channel(100); let (write_sender, write_receiver) = mpsc::channel(100); let store = MapIndexStore::new(path); - let actor = IndexActor::new(read_receiver, write_receiver, store); + let actor = IndexActor::new(read_receiver, write_receiver, store)?; tokio::task::spawn(actor.run()); - Self { + Ok(Self { read_sender, write_sender, - } + }) } - pub async fn create_index( - &self, - uuid: Uuid, - primary_key: Option, - ) -> Result { + pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { ret, @@ -357,11 +366,15 @@ impl IndexActorHandle { receiver.await.expect("IndexActor has been killed") } - pub async fn update(&self, meta: Processing, data: std::fs::File) -> UpdateResult { + pub async fn update( + &self, + meta: Processing, + data: std::fs::File, + ) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Update { ret, meta, data }; let _ = self.read_sender.send(msg).await; - receiver.await.expect("IndexActor has been killed") + Ok(receiver.await.expect("IndexActor has been killed")?) } pub async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { @@ -441,7 +454,7 @@ impl IndexStore for MapIndexStore { let meta = match self.meta_store.write().await.entry(uuid.clone()) { Entry::Vacant(entry) => { let now = Utc::now(); - let meta = IndexMeta{ + let meta = IndexMeta { uuid, created_at: now.clone(), updated_at: now, @@ -478,7 +491,7 @@ impl IndexStore for MapIndexStore { Entry::Vacant(index_entry) => match self.meta_store.write().await.entry(uuid.clone()) { Entry::Vacant(meta_entry) => { let now = Utc::now(); - let meta = IndexMeta{ + let meta = IndexMeta { uuid, created_at: now.clone(), updated_at: now, @@ -491,12 +504,13 @@ impl IndexStore for MapIndexStore { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &db_path) + .map_err(|e| IndexError::Error(e))?; let index = Index(Arc::new(index)); Ok(index) }) .await - .expect("thread died"); + .expect("thread died"); Ok(index_entry.insert(index?).clone()) } @@ -508,12 +522,13 @@ impl IndexStore for MapIndexStore { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &db_path) + .map_err(|e| IndexError::Error(e))?; let index = Index(Arc::new(index)); Ok(index) }) .await - .expect("thread died"); + .expect("thread died"); Ok(index_entry.insert(index?).clone()) } @@ -540,11 +555,14 @@ impl IndexStore for MapIndexStore { } async fn update_index(&self, uuid: Uuid, f: F) -> Result - where F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static, + where + F: FnOnce(Index) -> Result + Send + Sync + 'static, + R: Sync + Send + 'static, { let index = self.get_or_create(uuid.clone(), None).await?; - let mut meta = self.get_meta(&uuid).await? + let mut meta = self + .get_meta(&uuid) + .await? .ok_or(IndexError::UnexistingIndex)?; match f(index) { Ok(r) => { @@ -552,7 +570,7 @@ impl IndexStore for MapIndexStore { self.meta_store.write().await.insert(uuid, meta); Ok(r) } - Err(e) => Err(e) + Err(e) => Err(e), } } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index bcf663ec8..0bae7f42d 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -69,11 +69,11 @@ enum IndexControllerMsg { } impl IndexController { - pub fn new(path: impl AsRef) -> Self { + pub fn new(path: impl AsRef) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); - let index_actor = index_actor::IndexActorHandle::new(&path); + let index_actor = index_actor::IndexActorHandle::new(&path)?; let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); - Self { uuid_resolver, index_handle: index_actor, update_handle } + Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) } pub async fn add_documents( diff --git a/src/index_controller/update_store.rs b/src/index_controller/update_store.rs index 02c15ed8c..b8f174a34 100644 --- a/src/index_controller/update_store.rs +++ b/src/index_controller/update_store.rs @@ -30,18 +30,18 @@ pub trait HandleUpdate { &mut self, meta: Processing, content: File, - ) -> Result, Failed>; + ) -> anyhow::Result, Failed>>; } impl HandleUpdate for F where - F: FnMut(Processing, File) -> Result, Failed>, + F: FnMut(Processing, File) -> anyhow::Result, Failed>>, { fn handle_update( &mut self, meta: Processing, content: File, - ) -> Result, Failed> { + ) -> anyhow::Result, Failed>> { self(meta, content) } } @@ -100,7 +100,7 @@ where update_store.process_pending_update(handler) }) .await - .unwrap(); + .expect("Fatal error processing update."); match res { Ok(Some(_)) => (), Ok(None) => break, @@ -185,7 +185,7 @@ where /// Executes the user provided function on the next pending update (the one with the lowest id). /// This is asynchronous as it let the user process the update with a read-only txn and /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, mut handler: U) -> heed::Result> + fn process_pending_update(&self, mut handler: U) -> anyhow::Result> where U: HandleUpdate, { @@ -209,7 +209,7 @@ where self.processing.write().unwrap().replace(processing.clone()); let file = File::open(&content_path)?; // Process the pending update using the provided user function. - let result = handler.handle_update(processing, file); + let result = handler.handle_update(processing, file)?; drop(rtxn); // Once the pending update have been successfully processed From 4e1597bd1dbbc9c1b10a85124f9a748be6031e26 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 16:27:29 +0100 Subject: [PATCH 146/522] clean Uuid resolver actor --- src/index_controller/mod.rs | 38 ++++------- src/index_controller/uuid_resolver.rs | 93 +++++++++++++++++---------- 2 files changed, 74 insertions(+), 57 deletions(-) diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0bae7f42d..a824852bf 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -10,7 +10,6 @@ use std::sync::Arc; use std::time::Duration; use actix_web::web::{Bytes, Payload}; -use anyhow::Context; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; @@ -108,7 +107,7 @@ impl IndexController { } pub async fn clear_documents(&self, index: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await?.unwrap(); + let uuid = self.uuid_resolver.resolve(index).await?; let meta = UpdateMeta::ClearDocuments; let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; @@ -116,7 +115,7 @@ impl IndexController { } pub async fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await.unwrap().unwrap(); + let uuid = self.uuid_resolver.resolve(index).await?; let meta = UpdateMeta::DeleteDocuments; let (sender, receiver) = mpsc::channel(10); @@ -153,8 +152,7 @@ impl IndexController { pub async fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { let uuid = self.uuid_resolver .delete(index_uid) - .await? - .context("index not found")?; + .await?; self.update_handle.delete(uuid.clone()).await?; self.index_handle.delete(uuid).await?; Ok(()) @@ -163,16 +161,14 @@ impl IndexController { pub async fn update_status(&self, index: String, id: u64) -> anyhow::Result> { let uuid = self.uuid_resolver .resolve(index) - .await? - .context("index not found")?; + .await?; let result = self.update_handle.update_status(uuid, id).await?; Ok(result) } pub async fn all_update_status(&self, index: String) -> anyhow::Result> { let uuid = self.uuid_resolver - .resolve(index).await? - .context("index not found")?; + .resolve(index).await?; let result = self.update_handle.get_all_updates_status(uuid).await?; Ok(result) } @@ -195,8 +191,7 @@ impl IndexController { pub async fn settings(&self, index: String) -> anyhow::Result { let uuid = self.uuid_resolver .resolve(index.clone()) - .await? - .with_context(|| format!("Index {:?} doesn't exist", index))?; + .await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) } @@ -210,8 +205,7 @@ impl IndexController { ) -> anyhow::Result> { let uuid = self.uuid_resolver .resolve(index.clone()) - .await? - .with_context(|| format!("Index {:?} doesn't exist", index))?; + .await?; let documents = self.index_handle.documents(uuid, offset, limit, attributes_to_retrieve).await?; Ok(documents) } @@ -224,8 +218,7 @@ impl IndexController { ) -> anyhow::Result { let uuid = self.uuid_resolver .resolve(index.clone()) - .await? - .with_context(|| format!("Index {:?} doesn't exist", index))?; + .await?; let document = self.index_handle.document(uuid, doc_id, attributes_to_retrieve).await?; Ok(document) } @@ -235,21 +228,18 @@ impl IndexController { } pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); + let uuid = self.uuid_resolver.resolve(name).await?; let result = self.index_handle.search(uuid, query).await?; Ok(result) } pub async fn get_index(&self, name: String) -> anyhow::Result> { let uuid = self.uuid_resolver.resolve(name.clone()).await?; - if let Some(uuid) = uuid { - let result = self.index_handle - .get_index_meta(uuid) - .await? - .map(|meta| IndexMetadata { name, meta }); - return Ok(result) - } - Ok(None) + let result = self.index_handle + .get_index_meta(uuid) + .await? + .map(|meta| IndexMetadata { name, meta }); + Ok(result) } } diff --git a/src/index_controller/uuid_resolver.rs b/src/index_controller/uuid_resolver.rs index e2539fdb2..8d33edef4 100644 --- a/src/index_controller/uuid_resolver.rs +++ b/src/index_controller/uuid_resolver.rs @@ -1,10 +1,10 @@ -use thiserror::Error; -use tokio::sync::{RwLock, mpsc, oneshot}; -use uuid::Uuid; +use log::{info, warn}; +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::sync::Arc; -use std::collections::hash_map::Entry; -use log::{info, warn}; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot, RwLock}; +use uuid::Uuid; pub type Result = std::result::Result; @@ -12,7 +12,7 @@ pub type Result = std::result::Result; enum UuidResolveMsg { Resolve { name: String, - ret: oneshot::Sender>>, + ret: oneshot::Sender>, }, GetOrCreate { name: String, @@ -24,7 +24,7 @@ enum UuidResolveMsg { }, Delete { name: String, - ret: oneshot::Sender>>, + ret: oneshot::Sender>, }, List { ret: oneshot::Sender>>, @@ -46,13 +46,20 @@ impl UuidResolverActor { info!("uuid resolver started"); - // TODO: benchmark and use buffered streams to improve throughput. loop { match self.inbox.recv().await { - Some(Create { name, ret }) => self.handle_create(name, ret).await, - Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, - Some(Resolve { name, ret }) => self.handle_resolve(name, ret).await, - Some(Delete { name, ret }) => self.handle_delete(name, ret).await, + Some(Create { name, ret }) => { + let _ = ret.send(self.handle_create(name).await); + } + Some(GetOrCreate { name, ret }) => { + let _ = ret.send(self.handle_get_or_create(name).await); + } + Some(Resolve { name, ret }) => { + let _ = ret.send(self.handle_resolve(name).await); + } + Some(Delete { name, ret }) => { + let _ = ret.send(self.handle_delete(name).await); + } Some(List { ret }) => { let _ = ret.send(self.handle_list().await); } @@ -64,24 +71,26 @@ impl UuidResolverActor { warn!("exiting uuid resolver loop"); } - async fn handle_create(&self, name: String, ret: oneshot::Sender>) { - let result = self.store.create_uuid(name, true).await; - let _ = ret.send(result); + async fn handle_create(&self, name: String) -> Result { + self.store.create_uuid(name, true).await } - async fn handle_get_or_create(&self, name: String, ret: oneshot::Sender>) { - let result = self.store.create_uuid(name, false).await; - let _ = ret.send(result); + async fn handle_get_or_create(&self, name: String) -> Result { + self.store.create_uuid(name, false).await } - async fn handle_resolve(&self, name: String, ret: oneshot::Sender>>) { - let result = self.store.get_uuid(&name).await; - let _ = ret.send(result); + async fn handle_resolve(&self, name: String) -> Result { + self.store + .get_uuid(&name) + .await? + .ok_or(UuidError::UnexistingIndex(name)) } - async fn handle_delete(&self, name: String, ret: oneshot::Sender>>) { - let result = self.store.delete(&name).await; - let _ = ret.send(result); + async fn handle_delete(&self, name: String) -> Result { + self.store + .delete(&name) + .await? + .ok_or(UuidError::UnexistingIndex(name)) } async fn handle_list(&self) -> Result> { @@ -104,39 +113,49 @@ impl UuidResolverHandle { Self { sender } } - pub async fn resolve(&self, name: String) -> anyhow::Result> { + pub async fn resolve(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Resolve { name, ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } pub async fn get_or_create(&self, name: String) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::GetOrCreate { name, ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } pub async fn create(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Create { name, ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } - pub async fn delete(&self, name: String) -> anyhow::Result> { + pub async fn delete(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Delete { name, ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } pub async fn list(&self) -> anyhow::Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::List { ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } } @@ -144,6 +163,8 @@ impl UuidResolverHandle { pub enum UuidError { #[error("Name already exist.")] NameAlreadyExist, + #[error("Index \"{0}\" doesn't exist.")] + UnexistingIndex(String), } #[async_trait::async_trait] @@ -168,7 +189,7 @@ impl UuidStore for MapUuidStore { } else { Ok(entry.get().clone()) } - }, + } Entry::Vacant(entry) => { let uuid = Uuid::new_v4(); let uuid = entry.insert(uuid); @@ -186,7 +207,13 @@ impl UuidStore for MapUuidStore { } async fn list(&self) -> Result> { - let list = self.0.read().await.iter().map(|(name, uuid)| (name.to_owned(), uuid.clone())).collect(); + let list = self + .0 + .read() + .await + .iter() + .map(|(name, uuid)| (name.to_owned(), uuid.clone())) + .collect(); Ok(list) } } From 4a0f5f1b03585f8511e840717fe1ef1bae446e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 8 Mar 2021 21:22:30 +0100 Subject: [PATCH 147/522] Make sure that we do not use jemalloc on macos --- meilisearch-http/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 58faaebd1..90a076849 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -76,5 +76,5 @@ urlencoding = "1.1.1" [features] default = ["sentry"] -[target.'cfg(unix)'.dependencies] +[target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" From 562da9dd3fb06caf5eb4b0f2049f7926d7150119 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 11:56:51 +0100 Subject: [PATCH 148/522] fix test compilation --- Cargo.lock | 71 +++++------ Cargo.toml | 10 +- src/helpers/mod.rs | 4 +- src/index_controller/update_store.rs | 169 +++++++++++++-------------- src/lib.rs | 53 +++++++++ src/main.rs | 54 +-------- tests/common/index.rs | 4 +- tests/common/service.rs | 14 +-- 8 files changed, 180 insertions(+), 199 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbae20c98..0bdc739d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.0.0-beta.3" +version = "3.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a12706e793a92377f85cec219514b72625b3b89f9b4912d8bfb53ab6a615bf0" +checksum = "8a01f9e0681608afa887d4269a0857ac4226f09ba5ceda25939e8391c9da610a" dependencies = [ "actix-codec 0.4.0-beta.1", "actix-rt 2.1.0", @@ -127,22 +127,21 @@ dependencies = [ "brotli2", "bytes 1.0.1", "bytestring", + "cfg-if 1.0.0", "cookie", "derive_more", "encoding_rs", "flate2", - "futures-channel", "futures-core", "futures-util", - "h2 0.3.0", + "h2 0.3.1", "http", "httparse", - "indexmap", "itoa", "language-tags", - "lazy_static", "log", "mime", + "once_cell", "percent-encoding", "pin-project 1.0.2", "rand 0.8.3", @@ -151,9 +150,9 @@ dependencies = [ "serde_json", "serde_urlencoded", "sha-1 0.9.2", - "slab", "smallvec", "time 0.2.23", + "tokio 1.2.0", ] [[package]] @@ -324,12 +323,12 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.0.0-beta.3" +version = "4.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9683dc8c3037ea524e0fec6032d34e1cb1ee72c4eb8689f428a60c2a544ea3" +checksum = "1d95e50c9e32e8456220b5804867de76e97a86ab8c38b51c9edcccc0f0fddca7" dependencies = [ "actix-codec 0.4.0-beta.1", - "actix-http 3.0.0-beta.3", + "actix-http 3.0.0-beta.4", "actix-macros 0.2.0", "actix-router", "actix-rt 2.1.0", @@ -362,9 +361,9 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "0.5.0-beta.1" +version = "0.5.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8313dc4cbcae1785a7f14c3dfb7dfeb25fe96a03b20e5c38fe026786def5aa70" +checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ "proc-macro2", "quote", @@ -500,12 +499,12 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "awc" -version = "3.0.0-beta.2" +version = "3.0.0-beta.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7225ad81fbad09ef56ccc61e0688abe8494a68722c5d0df5e2fc8b724a200b" +checksum = "09aecd8728f6491a62b27454ea4b36fb7e50faf32928b0369b644e402c651f4e" dependencies = [ "actix-codec 0.4.0-beta.1", - "actix-http 3.0.0-beta.3", + "actix-http 3.0.0-beta.4", "actix-rt 2.1.0", "actix-service 2.0.0-beta.4", "base64 0.13.0", @@ -513,9 +512,11 @@ dependencies = [ "cfg-if 1.0.0", "derive_more", "futures-core", + "itoa", "log", "mime", "percent-encoding", + "pin-project-lite 0.2.0", "rand 0.8.3", "rustls 0.19.0", "serde", @@ -1310,9 +1311,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5" +checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" dependencies = [ "bytes 1.0.1", "fnv", @@ -1325,7 +1326,6 @@ dependencies = [ "tokio 1.2.0", "tokio-util 0.6.3", "tracing", - "tracing-futures", ] [[package]] @@ -1761,11 +1761,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] -name = "meilisearch" +name = "meilisearch-error" +version = "0.19.0" +dependencies = [ + "actix-http 2.2.0", +] + +[[package]] +name = "meilisearch-http" version = "0.17.0" dependencies = [ "actix-cors", - "actix-service 1.0.6", + "actix-http 3.0.0-beta.4", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", "actix-web", "anyhow", "assert-json-diff", @@ -1813,19 +1822,11 @@ dependencies = [ "tempdir", "tempfile", "thiserror", - "tokio 0.2.24", "tokio 1.2.0", "uuid", "vergen", ] -[[package]] -name = "meilisearch-error" -version = "0.19.0" -dependencies = [ - "actix-http 2.2.0", -] - [[package]] name = "meilisearch-tokenizer" version = "0.1.1" @@ -3236,7 +3237,6 @@ dependencies = [ "pin-project-lite 0.1.11", "signal-hook-registry", "slab", - "tokio-macros 0.2.6", "winapi 0.3.9", ] @@ -3256,21 +3256,10 @@ dependencies = [ "parking_lot", "pin-project-lite 0.2.0", "signal-hook-registry", - "tokio-macros 1.1.0", + "tokio-macros", "winapi 0.3.9", ] -[[package]] -name = "tokio-macros" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tokio-macros" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 9bc5e4d1d..792ebd763 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Quentin de Quelen ", "Clément Renault HandleUpdate for F - where - F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static, - { - fn handle_update( - &mut self, - meta: Processing, - content: &[u8], - ) -> Result, Failed> { - self(meta, content) - } - } + //#[test] + //fn simple() { + //let dir = tempfile::tempdir().unwrap(); + //let mut options = EnvOpenOptions::new(); + //options.map_size(4096 * 100); + //let update_store = UpdateStore::open( + //options, + //dir, + //|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + //let new_meta = meta.meta().to_string() + " processed"; + //let processed = meta.process(new_meta); + //Ok(processed) + //}, + //) + //.unwrap(); - #[test] - fn simple() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open( - options, - dir, - |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }, - ) - .unwrap(); + //let meta = String::from("kiki"); + //let update = update_store.register_update(meta, &[]).unwrap(); + //thread::sleep(Duration::from_millis(100)); + //let meta = update_store.meta(update.id()).unwrap().unwrap(); + //if let UpdateStatus::Processed(Processed { success, .. }) = meta { + //assert_eq!(success, "kiki processed"); + //} else { + //panic!() + //} + //} - let meta = String::from("kiki"); - let update = update_store.register_update(meta, &[]).unwrap(); - thread::sleep(Duration::from_millis(100)); - let meta = update_store.meta(update.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - } + //#[test] + //#[ignore] + //fn long_running_update() { + //let dir = tempfile::tempdir().unwrap(); + //let mut options = EnvOpenOptions::new(); + //options.map_size(4096 * 100); + //let update_store = UpdateStore::open( + //options, + //dir, + //|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + //thread::sleep(Duration::from_millis(400)); + //let new_meta = meta.meta().to_string() + "processed"; + //let processed = meta.process(new_meta); + //Ok(processed) + //}, + //) + //.unwrap(); - #[test] - #[ignore] - fn long_running_update() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open( - options, - dir, - |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }, - ) - .unwrap(); + //let before_register = Instant::now(); - let before_register = Instant::now(); + //let meta = String::from("kiki"); + //let update_kiki = update_store.register_update(meta, &[]).unwrap(); + //assert!(before_register.elapsed() < Duration::from_millis(200)); - let meta = String::from("kiki"); - let update_kiki = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); + //let meta = String::from("coco"); + //let update_coco = update_store.register_update(meta, &[]).unwrap(); + //assert!(before_register.elapsed() < Duration::from_millis(200)); - let meta = String::from("coco"); - let update_coco = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); + //let meta = String::from("cucu"); + //let update_cucu = update_store.register_update(meta, &[]).unwrap(); + //assert!(before_register.elapsed() < Duration::from_millis(200)); - let meta = String::from("cucu"); - let update_cucu = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); + //thread::sleep(Duration::from_millis(400 * 3 + 100)); - thread::sleep(Duration::from_millis(400 * 3 + 100)); + //let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + //if let UpdateStatus::Processed(Processed { success, .. }) = meta { + //assert_eq!(success, "kiki processed"); + //} else { + //panic!() + //} - let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } + //let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + //if let UpdateStatus::Processed(Processed { success, .. }) = meta { + //assert_eq!(success, "coco processed"); + //} else { + //panic!() + //} - let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "coco processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "cucu processed"); - } else { - panic!() - } - } -} + //let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + //if let UpdateStatus::Processed(Processed { success, .. }) = meta { + //assert_eq!(success, "cucu processed"); + //} else { + //panic!() + //} + //} +//} diff --git a/src/lib.rs b/src/lib.rs index 32c8e4266..bd7379d7c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,3 +13,56 @@ mod index_controller; pub use option::Opt; pub use self::data::Data; + +#[macro_export] +macro_rules! create_app { + ($data:expr, $enable_frontend:expr) => { + { + use actix_cors::Cors; + use actix_web::App; + use actix_web::middleware::TrailingSlash; + use actix_web::{web, middleware}; + use meilisearch_http::error::payload_error_handler; + use meilisearch_http::routes::*; + + let app = App::new() + .data($data.clone()) + .app_data( + web::JsonConfig::default() + .limit($data.http_payload_size_limit()) + .content_type(|_mime| true) // Accept all mime types + .error_handler(|err, _req| payload_error_handler(err).into()), + ) + .app_data( + web::QueryConfig::default() + .error_handler(|err, _req| payload_error_handler(err).into()) + ) + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(stop_words::services) + .configure(synonym::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services); + //.configure(routes::dump::services); + let app = if $enable_frontend { + app + .service(load_html) + .service(load_css) + } else { + app + }; + app.wrap( + Cors::default() + .send_wildcard() + .allowed_headers(vec!["content-type", "x-meili-api-key"]) + .max_age(86_400) // 24h + ) + .wrap(middleware::Logger::default()) + .wrap(middleware::Compress::default()) + .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) + } + }; +} diff --git a/src/main.rs b/src/main.rs index 9e69c4091..36959c5b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,13 +1,9 @@ use std::env; -use actix_cors::Cors; -use actix_web::{middleware, HttpServer, web, web::ServiceConfig}; +use actix_web::HttpServer; use main_error::MainError; -use meilisearch::{Data, Opt}; +use meilisearch_http::{Data, Opt, create_app}; use structopt::StructOpt; -use actix_web::App; -use meilisearch::error::payload_error_handler; -use actix_web::middleware::TrailingSlash; //mod analytics; @@ -83,38 +79,8 @@ async fn main() -> Result<(), MainError> { } async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box> { - use meilisearch::routes::*; - let http_server = HttpServer::new(move || { - let app = App::new() - .configure(|c| configure_data(c, &data)) - .configure(document::services) - .configure(index::services) - .configure(search::services) - .configure(settings::services) - .configure(stop_words::services) - .configure(synonym::services) - .configure(health::services) - .configure(stats::services) - .configure(key::services); - //.configure(routes::dump::services); - let app = if enable_frontend { - app - .service(meilisearch::routes::load_html) - .service(meilisearch::routes::load_css) - } else { - app - }; - app.wrap( - Cors::default() - .send_wildcard() - .allowed_headers(vec!["content-type", "x-meili-api-key"]) - .max_age(86_400) // 24h - ) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - }) + let http_server = HttpServer::new(move || create_app!(&data, enable_frontend)) // Disable signals allows the server to terminate immediately when a user enter CTRL-C .disable_signals(); @@ -129,20 +95,6 @@ async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box Ok(()) } -fn configure_data(config: &mut ServiceConfig, data: &Data) { - config - .data(data.clone()) - .app_data( - web::JsonConfig::default() - .limit(data.http_payload_size_limit()) - .content_type(|_mime| true) // Accept all mime types - .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .app_data( - web::QueryConfig::default() - .error_handler(|err, _req| payload_error_handler(err).into()) - ); -} pub fn print_launch_resume(opt: &Opt, data: &Data) { let ascii_name = r#" diff --git a/tests/common/index.rs b/tests/common/index.rs index 710b96a9f..58a8de200 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -2,7 +2,7 @@ use std::time::Duration; use actix_web::http::StatusCode; use serde_json::{json, Value}; -use tokio::time::delay_for; +use tokio::time::sleep; use super::service::Service; @@ -79,7 +79,7 @@ impl Index<'_> { return response; } - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; } panic!("Timeout waiting for update id"); } diff --git a/tests/common/service.rs b/tests/common/service.rs index 8e797c00d..feff49cad 100644 --- a/tests/common/service.rs +++ b/tests/common/service.rs @@ -2,14 +2,14 @@ use actix_web::{http::StatusCode, test}; use serde_json::Value; use meilisearch_http::data::Data; -use meilisearch_http::helpers::NormalizePath; +use meilisearch_http::create_app; pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -26,12 +26,12 @@ impl Service { /// Send a test post request from a text body, with a `content-type:application/json` header. pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) - .header("content-type", "application/json") + .insert_header(("content-type", "application/json")) .to_request(); let res = test::call_service(&mut app, req).await; let status_code = res.status(); @@ -43,7 +43,7 @@ impl Service { pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; @@ -56,7 +56,7 @@ impl Service { pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::put() .uri(url.as_ref()) @@ -72,7 +72,7 @@ impl Service { pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; From 8061a04661017b8b0764ecd361c51920af824f01 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 13:38:30 +0100 Subject: [PATCH 149/522] add test assets --- tests/assets/test_set.json | 1613 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1613 insertions(+) create mode 100644 tests/assets/test_set.json diff --git a/tests/assets/test_set.json b/tests/assets/test_set.json new file mode 100644 index 000000000..63534c896 --- /dev/null +++ b/tests/assets/test_set.json @@ -0,0 +1,1613 @@ +[ + { + "id": 0, + "isActive": false, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ] + }, + { + "id": 1, + "isActive": true, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ] + }, + { + "id": 2, + "isActive": true, + "balance": "$2,467.47", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", + "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", + "registered": "2014-10-28T12:59:30 -01:00", + "latitude": -64.008555, + "longitude": 11.867098, + "tags": [ + "good first issue" + ] + }, + { + "id": 3, + "isActive": true, + "balance": "$3,344.40", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Adeline Flynn", + "gender": "female", + "email": "adelineflynn@chorizon.com", + "phone": "+1 (994) 600-2840", + "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948", + "about": "Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n", + "registered": "2014-03-27T06:24:45 -01:00", + "latitude": -74.485173, + "longitude": -11.059859, + "tags": [ + "bug", + "good first issue", + "wontfix", + "new issue" + ] + }, + { + "id": 4, + "isActive": false, + "balance": "$2,575.78", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "Green", + "name": "Mariana Pacheco", + "gender": "female", + "email": "marianapacheco@chorizon.com", + "phone": "+1 (820) 414-2223", + "address": "664 Rapelye Street, Faywood, California, 7320", + "about": "Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n", + "registered": "2015-09-02T03:23:35 -02:00", + "latitude": 75.763501, + "longitude": -78.777124, + "tags": [ + "new issue" + ] + }, + { + "id": 5, + "isActive": true, + "balance": "$3,793.09", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "Green", + "name": "Warren Watson", + "gender": "male", + "email": "warrenwatson@chorizon.com", + "phone": "+1 (807) 583-2427", + "address": "671 Prince Street, Faxon, Connecticut, 4275", + "about": "Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n", + "registered": "2017-06-04T06:02:17 -02:00", + "latitude": 29.979223, + "longitude": 25.358943, + "tags": [ + "wontfix", + "wontfix", + "wontfix" + ] + }, + { + "id": 6, + "isActive": true, + "balance": "$2,919.70", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "blue", + "name": "Shelia Berry", + "gender": "female", + "email": "sheliaberry@chorizon.com", + "phone": "+1 (853) 511-2651", + "address": "437 Forrest Street, Coventry, Illinois, 2056", + "about": "Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n", + "registered": "2018-07-11T02:45:01 -02:00", + "latitude": 54.815991, + "longitude": -118.690609, + "tags": [ + "good first issue", + "bug", + "wontfix", + "new issue" + ] + }, + { + "id": 7, + "isActive": true, + "balance": "$1,349.50", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Chrystal Boyd", + "gender": "female", + "email": "chrystalboyd@chorizon.com", + "phone": "+1 (936) 563-2802", + "address": "670 Croton Loop, Sussex, Florida, 4692", + "about": "Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n", + "registered": "2016-11-01T07:36:04 -01:00", + "latitude": -24.711933, + "longitude": 147.246705, + "tags": [] + }, + { + "id": 8, + "isActive": false, + "balance": "$3,999.56", + "picture": "http://placehold.it/32x32", + "age": 30, + "color": "brown", + "name": "Martin Porter", + "gender": "male", + "email": "martinporter@chorizon.com", + "phone": "+1 (895) 580-2304", + "address": "577 Regent Place, Aguila, Guam, 6554", + "about": "Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n", + "registered": "2014-09-20T02:08:30 -02:00", + "latitude": -88.344273, + "longitude": 37.964466, + "tags": [] + }, + { + "id": 9, + "isActive": true, + "balance": "$3,729.71", + "picture": "http://placehold.it/32x32", + "age": 26, + "color": "blue", + "name": "Kelli Mendez", + "gender": "female", + "email": "kellimendez@chorizon.com", + "phone": "+1 (936) 401-2236", + "address": "242 Caton Place, Grazierville, Alabama, 3968", + "about": "Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n", + "registered": "2018-05-04T10:35:30 -02:00", + "latitude": 49.37551, + "longitude": 41.872323, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 10, + "isActive": false, + "balance": "$1,127.47", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "blue", + "name": "Maddox Johns", + "gender": "male", + "email": "maddoxjohns@chorizon.com", + "phone": "+1 (892) 470-2357", + "address": "756 Beard Street, Avalon, Louisiana, 114", + "about": "Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n", + "registered": "2016-04-22T06:41:25 -02:00", + "latitude": 66.640229, + "longitude": -17.222666, + "tags": [ + "new issue", + "good first issue", + "good first issue", + "new issue" + ] + }, + { + "id": 11, + "isActive": true, + "balance": "$1,351.43", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Evans Wagner", + "gender": "male", + "email": "evanswagner@chorizon.com", + "phone": "+1 (889) 496-2332", + "address": "118 Monaco Place, Lutsen, Delaware, 6209", + "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", + "registered": "2016-10-27T01:26:31 -02:00", + "latitude": -77.673222, + "longitude": -142.657214, + "tags": [ + "good first issue", + "good first issue" + ] + }, + { + "id": 12, + "isActive": false, + "balance": "$3,394.96", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "blue", + "name": "Aida Kirby", + "gender": "female", + "email": "aidakirby@chorizon.com", + "phone": "+1 (942) 532-2325", + "address": "797 Engert Avenue, Wilsonia, Idaho, 6532", + "about": "Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n", + "registered": "2018-06-18T04:39:57 -02:00", + "latitude": -58.062041, + "longitude": 34.999254, + "tags": [ + "new issue", + "wontfix", + "bug", + "new issue" + ] + }, + { + "id": 13, + "isActive": true, + "balance": "$2,812.62", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "blue", + "name": "Nelda Burris", + "gender": "female", + "email": "neldaburris@chorizon.com", + "phone": "+1 (813) 600-2576", + "address": "160 Opal Court, Fowlerville, Tennessee, 2170", + "about": "Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n", + "registered": "2015-08-15T12:39:53 -02:00", + "latitude": 66.6871, + "longitude": 179.549488, + "tags": [ + "wontfix" + ] + }, + { + "id": 14, + "isActive": true, + "balance": "$1,718.33", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Jennifer Hart", + "gender": "female", + "email": "jenniferhart@chorizon.com", + "phone": "+1 (850) 537-2513", + "address": "124 Veranda Place, Nash, Utah, 985", + "about": "Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n", + "registered": "2016-09-04T11:46:59 -02:00", + "latitude": -66.827751, + "longitude": 99.220079, + "tags": [ + "wontfix", + "bug", + "new issue", + "new issue" + ] + }, + { + "id": 15, + "isActive": false, + "balance": "$2,698.16", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "blue", + "name": "Aurelia Contreras", + "gender": "female", + "email": "aureliacontreras@chorizon.com", + "phone": "+1 (932) 442-3103", + "address": "655 Dwight Street, Grapeview, Palau, 8356", + "about": "Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n", + "registered": "2014-09-11T10:43:15 -02:00", + "latitude": -71.328973, + "longitude": 133.404895, + "tags": [ + "wontfix", + "bug", + "good first issue" + ] + }, + { + "id": 16, + "isActive": true, + "balance": "$3,303.25", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Estella Bass", + "gender": "female", + "email": "estellabass@chorizon.com", + "phone": "+1 (825) 436-2909", + "address": "435 Rockwell Place, Garberville, Wisconsin, 2230", + "about": "Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n", + "registered": "2017-11-23T09:32:09 -01:00", + "latitude": 81.17014, + "longitude": -145.262693, + "tags": [ + "new issue" + ] + }, + { + "id": 17, + "isActive": false, + "balance": "$3,579.20", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "brown", + "name": "Ortega Brennan", + "gender": "male", + "email": "ortegabrennan@chorizon.com", + "phone": "+1 (906) 526-2287", + "address": "440 Berry Street, Rivera, Maine, 1849", + "about": "Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n", + "registered": "2016-03-31T02:17:13 -02:00", + "latitude": -68.407524, + "longitude": -113.642067, + "tags": [ + "new issue", + "wontfix" + ] + }, + { + "id": 18, + "isActive": false, + "balance": "$1,484.92", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "blue", + "name": "Leonard Tillman", + "gender": "male", + "email": "leonardtillman@chorizon.com", + "phone": "+1 (864) 541-3456", + "address": "985 Provost Street, Charco, New Hampshire, 8632", + "about": "Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n", + "registered": "2018-05-06T08:21:27 -02:00", + "latitude": -8.581801, + "longitude": -61.910062, + "tags": [ + "wontfix", + "new issue", + "bug", + "bug" + ] + }, + { + "id": 19, + "isActive": true, + "balance": "$3,572.55", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Dale Payne", + "gender": "male", + "email": "dalepayne@chorizon.com", + "phone": "+1 (814) 469-3499", + "address": "536 Dare Court, Ironton, Arkansas, 8605", + "about": "Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n", + "registered": "2019-10-11T01:01:33 -02:00", + "latitude": -18.280968, + "longitude": -126.091797, + "tags": [ + "bug", + "wontfix", + "wontfix", + "wontfix" + ] + }, + { + "id": 20, + "isActive": true, + "balance": "$1,986.48", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Florence Long", + "gender": "female", + "email": "florencelong@chorizon.com", + "phone": "+1 (972) 557-3858", + "address": "519 Hendrickson Street, Templeton, Hawaii, 2389", + "about": "Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n", + "registered": "2016-05-02T09:18:59 -02:00", + "latitude": -27.110866, + "longitude": -45.09445, + "tags": [] + }, + { + "id": 21, + "isActive": true, + "balance": "$1,440.09", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "blue", + "name": "Levy Whitley", + "gender": "male", + "email": "levywhitley@chorizon.com", + "phone": "+1 (911) 458-2411", + "address": "187 Thomas Street, Hachita, North Carolina, 2989", + "about": "Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n", + "registered": "2014-04-30T07:31:38 -02:00", + "latitude": -6.537315, + "longitude": 171.813536, + "tags": [ + "bug" + ] + }, + { + "id": 22, + "isActive": false, + "balance": "$2,938.57", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Bernard Mcfarland", + "gender": "male", + "email": "bernardmcfarland@chorizon.com", + "phone": "+1 (979) 442-3386", + "address": "409 Hall Street, Keyport, Federated States Of Micronesia, 7011", + "about": "Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n", + "registered": "2017-08-10T10:07:59 -02:00", + "latitude": 63.766795, + "longitude": 68.177069, + "tags": [] + }, + { + "id": 23, + "isActive": true, + "balance": "$1,678.49", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "brown", + "name": "Blanca Mcclain", + "gender": "female", + "email": "blancamcclain@chorizon.com", + "phone": "+1 (976) 439-2772", + "address": "176 Crooke Avenue, Valle, Virginia, 5373", + "about": "Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n", + "registered": "2015-10-12T11:57:28 -02:00", + "latitude": -8.944564, + "longitude": -150.711709, + "tags": [ + "bug", + "wontfix", + "good first issue" + ] + }, + { + "id": 24, + "isActive": true, + "balance": "$2,276.87", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Espinoza Ford", + "gender": "male", + "email": "espinozaford@chorizon.com", + "phone": "+1 (945) 429-3975", + "address": "137 Bowery Street, Itmann, District Of Columbia, 1864", + "about": "Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n", + "registered": "2014-03-26T02:16:08 -01:00", + "latitude": -37.137666, + "longitude": -51.811757, + "tags": [ + "wontfix", + "bug" + ] + }, + { + "id": 25, + "isActive": true, + "balance": "$3,973.43", + "picture": "http://placehold.it/32x32", + "age": 29, + "color": "Green", + "name": "Sykes Conley", + "gender": "male", + "email": "sykesconley@chorizon.com", + "phone": "+1 (851) 401-3916", + "address": "345 Grand Street, Woodlands, Missouri, 4461", + "about": "Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n", + "registered": "2015-09-12T06:03:56 -02:00", + "latitude": 67.282955, + "longitude": -64.341323, + "tags": [ + "wontfix" + ] + }, + { + "id": 26, + "isActive": false, + "balance": "$1,431.50", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Barlow Duran", + "gender": "male", + "email": "barlowduran@chorizon.com", + "phone": "+1 (995) 436-2562", + "address": "481 Everett Avenue, Allison, Nebraska, 3065", + "about": "Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n", + "registered": "2017-06-29T04:28:43 -02:00", + "latitude": -38.70606, + "longitude": 55.02816, + "tags": [ + "new issue" + ] + }, + { + "id": 27, + "isActive": true, + "balance": "$3,478.27", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "blue", + "name": "Schwartz Morgan", + "gender": "male", + "email": "schwartzmorgan@chorizon.com", + "phone": "+1 (861) 507-2067", + "address": "451 Lincoln Road, Fairlee, Washington, 2717", + "about": "Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n", + "registered": "2016-05-10T08:34:54 -02:00", + "latitude": -75.886403, + "longitude": 93.044471, + "tags": [ + "bug", + "bug", + "wontfix", + "wontfix" + ] + }, + { + "id": 28, + "isActive": true, + "balance": "$2,825.59", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Kristy Leon", + "gender": "female", + "email": "kristyleon@chorizon.com", + "phone": "+1 (948) 465-2563", + "address": "594 Macon Street, Floris, South Dakota, 3565", + "about": "Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n", + "registered": "2014-12-14T04:10:29 -01:00", + "latitude": -50.01615, + "longitude": -68.908804, + "tags": [ + "wontfix", + "good first issue" + ] + }, + { + "id": 29, + "isActive": false, + "balance": "$3,028.03", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "blue", + "name": "Ashley Pittman", + "gender": "male", + "email": "ashleypittman@chorizon.com", + "phone": "+1 (928) 507-3523", + "address": "646 Adelphi Street, Clara, Colorado, 6056", + "about": "Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n", + "registered": "2016-01-07T10:40:48 -01:00", + "latitude": -58.766037, + "longitude": -124.828485, + "tags": [ + "wontfix" + ] + }, + { + "id": 30, + "isActive": true, + "balance": "$2,021.11", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Stacy Espinoza", + "gender": "female", + "email": "stacyespinoza@chorizon.com", + "phone": "+1 (999) 487-3253", + "address": "931 Alabama Avenue, Bangor, Alaska, 8215", + "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", + "registered": "2014-07-16T06:15:53 -02:00", + "latitude": 41.560197, + "longitude": 177.697, + "tags": [ + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 31, + "isActive": false, + "balance": "$3,609.82", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Vilma Garza", + "gender": "female", + "email": "vilmagarza@chorizon.com", + "phone": "+1 (944) 585-2021", + "address": "565 Tech Place, Sedley, Puerto Rico, 858", + "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", + "registered": "2017-06-30T07:43:52 -02:00", + "latitude": -12.574889, + "longitude": -54.771186, + "tags": [ + "new issue", + "wontfix", + "wontfix" + ] + }, + { + "id": 32, + "isActive": false, + "balance": "$2,882.34", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "brown", + "name": "June Dunlap", + "gender": "female", + "email": "junedunlap@chorizon.com", + "phone": "+1 (997) 504-2937", + "address": "353 Cozine Avenue, Goodville, Indiana, 1438", + "about": "Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n", + "registered": "2016-08-23T08:54:11 -02:00", + "latitude": -27.883363, + "longitude": -163.919683, + "tags": [ + "new issue", + "new issue", + "bug", + "wontfix" + ] + }, + { + "id": 33, + "isActive": true, + "balance": "$3,556.54", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Cecilia Greer", + "gender": "female", + "email": "ceciliagreer@chorizon.com", + "phone": "+1 (977) 573-3498", + "address": "696 Withers Street, Lydia, Oklahoma, 3220", + "about": "Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n", + "registered": "2017-01-13T11:30:12 -01:00", + "latitude": 60.467215, + "longitude": 84.684575, + "tags": [ + "wontfix", + "good first issue", + "good first issue", + "wontfix" + ] + }, + { + "id": 34, + "isActive": true, + "balance": "$1,413.35", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Mckay Schroeder", + "gender": "male", + "email": "mckayschroeder@chorizon.com", + "phone": "+1 (816) 480-3657", + "address": "958 Miami Court, Rehrersburg, Northern Mariana Islands, 567", + "about": "Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n", + "registered": "2016-02-08T04:50:15 -01:00", + "latitude": -72.413287, + "longitude": -159.254371, + "tags": [ + "good first issue" + ] + }, + { + "id": 35, + "isActive": true, + "balance": "$2,306.53", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Sawyer Mccormick", + "gender": "male", + "email": "sawyermccormick@chorizon.com", + "phone": "+1 (829) 569-3012", + "address": "749 Apollo Street, Eastvale, Texas, 7373", + "about": "Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n", + "registered": "2019-11-30T11:53:23 -01:00", + "latitude": -48.978194, + "longitude": 110.950191, + "tags": [ + "good first issue", + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 36, + "isActive": false, + "balance": "$1,844.54", + "picture": "http://placehold.it/32x32", + "age": 37, + "color": "brown", + "name": "Barbra Valenzuela", + "gender": "female", + "email": "barbravalenzuela@chorizon.com", + "phone": "+1 (992) 512-2649", + "address": "617 Schenck Court, Reinerton, Michigan, 2908", + "about": "Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n", + "registered": "2019-03-29T01:59:31 -01:00", + "latitude": 45.193723, + "longitude": -12.486778, + "tags": [ + "new issue", + "new issue", + "wontfix", + "wontfix" + ] + }, + { + "id": 37, + "isActive": false, + "balance": "$3,469.82", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "brown", + "name": "Opal Weiss", + "gender": "female", + "email": "opalweiss@chorizon.com", + "phone": "+1 (809) 400-3079", + "address": "535 Bogart Street, Frizzleburg, Arizona, 5222", + "about": "Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n", + "registered": "2019-09-04T07:22:28 -02:00", + "latitude": 72.50376, + "longitude": 61.656435, + "tags": [ + "bug", + "bug", + "good first issue", + "good first issue" + ] + }, + { + "id": 38, + "isActive": true, + "balance": "$1,992.38", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "Green", + "name": "Christina Short", + "gender": "female", + "email": "christinashort@chorizon.com", + "phone": "+1 (884) 589-2705", + "address": "594 Willmohr Street, Dexter, Montana, 660", + "about": "Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n", + "registered": "2014-01-21T09:31:56 -01:00", + "latitude": -42.762739, + "longitude": 77.052349, + "tags": [ + "bug", + "new issue" + ] + }, + { + "id": 39, + "isActive": false, + "balance": "$1,722.85", + "picture": "http://placehold.it/32x32", + "age": 29, + "color": "brown", + "name": "Golden Horton", + "gender": "male", + "email": "goldenhorton@chorizon.com", + "phone": "+1 (903) 426-2489", + "address": "191 Schenck Avenue, Mayfair, North Dakota, 5000", + "about": "Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n", + "registered": "2015-08-19T02:56:41 -02:00", + "latitude": 69.922534, + "longitude": 9.881433, + "tags": [ + "bug" + ] + }, + { + "id": 40, + "isActive": false, + "balance": "$1,656.54", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "blue", + "name": "Stafford Emerson", + "gender": "male", + "email": "staffordemerson@chorizon.com", + "phone": "+1 (992) 455-2573", + "address": "523 Thornton Street, Conway, Vermont, 6331", + "about": "Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n", + "registered": "2019-02-16T04:07:08 -01:00", + "latitude": -29.143111, + "longitude": -57.207703, + "tags": [ + "wontfix", + "good first issue", + "good first issue" + ] + }, + { + "id": 41, + "isActive": false, + "balance": "$1,861.56", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "brown", + "name": "Salinas Gamble", + "gender": "male", + "email": "salinasgamble@chorizon.com", + "phone": "+1 (901) 525-2373", + "address": "991 Nostrand Avenue, Kansas, Mississippi, 6756", + "about": "Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n", + "registered": "2017-08-21T05:47:53 -02:00", + "latitude": -22.593819, + "longitude": -63.613004, + "tags": [ + "good first issue", + "bug", + "bug", + "wontfix" + ] + }, + { + "id": 42, + "isActive": true, + "balance": "$3,179.74", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "brown", + "name": "Graciela Russell", + "gender": "female", + "email": "gracielarussell@chorizon.com", + "phone": "+1 (893) 464-3951", + "address": "361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713", + "about": "Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n", + "registered": "2015-05-18T09:52:56 -02:00", + "latitude": -14.634444, + "longitude": 12.931783, + "tags": [ + "wontfix", + "bug", + "wontfix" + ] + }, + { + "id": 43, + "isActive": true, + "balance": "$1,777.38", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "blue", + "name": "Arnold Bender", + "gender": "male", + "email": "arnoldbender@chorizon.com", + "phone": "+1 (945) 581-3808", + "address": "781 Lorraine Street, Gallina, American Samoa, 1832", + "about": "Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n", + "registered": "2018-12-23T02:26:30 -01:00", + "latitude": 41.208579, + "longitude": 51.948925, + "tags": [ + "bug", + "good first issue", + "good first issue", + "wontfix" + ] + }, + { + "id": 44, + "isActive": true, + "balance": "$2,893.45", + "picture": "http://placehold.it/32x32", + "age": 22, + "color": "Green", + "name": "Joni Spears", + "gender": "female", + "email": "jonispears@chorizon.com", + "phone": "+1 (916) 565-2124", + "address": "307 Harwood Place, Canterwood, Maryland, 2047", + "about": "Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n", + "registered": "2015-03-01T12:38:28 -01:00", + "latitude": 8.19071, + "longitude": 146.323808, + "tags": [ + "wontfix", + "new issue", + "good first issue", + "good first issue" + ] + }, + { + "id": 45, + "isActive": true, + "balance": "$2,830.36", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "brown", + "name": "Irene Bennett", + "gender": "female", + "email": "irenebennett@chorizon.com", + "phone": "+1 (904) 431-2211", + "address": "353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686", + "about": "Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n", + "registered": "2018-04-17T05:18:51 -02:00", + "latitude": -36.435177, + "longitude": -127.552573, + "tags": [ + "bug", + "wontfix" + ] + }, + { + "id": 46, + "isActive": true, + "balance": "$1,348.04", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "Green", + "name": "Lawson Curtis", + "gender": "male", + "email": "lawsoncurtis@chorizon.com", + "phone": "+1 (896) 532-2172", + "address": "942 Gerritsen Avenue, Southmont, Kansas, 8915", + "about": "Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n", + "registered": "2016-08-23T01:41:09 -02:00", + "latitude": -48.783539, + "longitude": 20.492944, + "tags": [] + }, + { + "id": 47, + "isActive": true, + "balance": "$1,132.41", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Goff May", + "gender": "male", + "email": "goffmay@chorizon.com", + "phone": "+1 (859) 453-3415", + "address": "225 Rutledge Street, Boonville, Massachusetts, 4081", + "about": "Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n", + "registered": "2014-10-25T07:32:30 -02:00", + "latitude": 13.079225, + "longitude": 76.215086, + "tags": [ + "bug" + ] + }, + { + "id": 48, + "isActive": true, + "balance": "$1,201.87", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Goodman Becker", + "gender": "male", + "email": "goodmanbecker@chorizon.com", + "phone": "+1 (825) 470-3437", + "address": "388 Seigel Street, Sisquoc, Kentucky, 8231", + "about": "Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n", + "registered": "2019-09-05T04:49:03 -02:00", + "latitude": -23.792094, + "longitude": -13.621221, + "tags": [ + "bug", + "bug", + "wontfix", + "new issue" + ] + }, + { + "id": 49, + "isActive": true, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ] + }, + { + "id": 50, + "isActive": true, + "balance": "$1,947.08", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "Green", + "name": "Guerra Mcintyre", + "gender": "male", + "email": "guerramcintyre@chorizon.com", + "phone": "+1 (951) 536-2043", + "address": "423 Lombardy Street, Stewart, West Virginia, 908", + "about": "Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n", + "registered": "2015-07-16T05:11:42 -02:00", + "latitude": 79.733743, + "longitude": -20.602356, + "tags": [ + "bug", + "good first issue", + "good first issue" + ] + }, + { + "id": 51, + "isActive": true, + "balance": "$2,960.90", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "blue", + "name": "Key Cervantes", + "gender": "male", + "email": "keycervantes@chorizon.com", + "phone": "+1 (931) 474-3865", + "address": "410 Barbey Street, Vernon, Oregon, 2328", + "about": "Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n", + "registered": "2019-12-15T12:13:35 -01:00", + "latitude": 47.627647, + "longitude": 117.049918, + "tags": [ + "new issue" + ] + }, + { + "id": 52, + "isActive": false, + "balance": "$1,884.02", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Karen Nelson", + "gender": "female", + "email": "karennelson@chorizon.com", + "phone": "+1 (993) 528-3607", + "address": "930 Frank Court, Dunbar, New York, 8810", + "about": "Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n", + "registered": "2014-06-23T09:21:44 -02:00", + "latitude": -59.059033, + "longitude": 76.565373, + "tags": [ + "new issue", + "bug" + ] + }, + { + "id": 53, + "isActive": true, + "balance": "$3,559.55", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "brown", + "name": "Caitlin Burnett", + "gender": "female", + "email": "caitlinburnett@chorizon.com", + "phone": "+1 (945) 480-2796", + "address": "516 Senator Street, Emory, Iowa, 4145", + "about": "In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n", + "registered": "2019-01-09T02:26:31 -01:00", + "latitude": -82.774237, + "longitude": 42.316194, + "tags": [ + "bug", + "good first issue" + ] + }, + { + "id": 54, + "isActive": true, + "balance": "$2,113.29", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Richards Walls", + "gender": "male", + "email": "richardswalls@chorizon.com", + "phone": "+1 (865) 517-2982", + "address": "959 Brightwater Avenue, Stevens, Nevada, 2968", + "about": "Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n", + "registered": "2014-09-25T06:51:22 -02:00", + "latitude": 80.09202, + "longitude": 87.49759, + "tags": [ + "wontfix", + "wontfix", + "bug" + ] + }, + { + "id": 55, + "isActive": true, + "balance": "$1,977.66", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "brown", + "name": "Combs Stanley", + "gender": "male", + "email": "combsstanley@chorizon.com", + "phone": "+1 (827) 419-2053", + "address": "153 Beverley Road, Siglerville, South Carolina, 3666", + "about": "Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n", + "registered": "2019-08-22T07:53:15 -02:00", + "latitude": 78.386181, + "longitude": 143.661058, + "tags": [] + }, + { + "id": 56, + "isActive": false, + "balance": "$3,886.12", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "brown", + "name": "Tucker Barry", + "gender": "male", + "email": "tuckerbarry@chorizon.com", + "phone": "+1 (808) 544-3433", + "address": "805 Jamaica Avenue, Cornfields, Minnesota, 3689", + "about": "Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n", + "registered": "2016-08-29T07:28:00 -02:00", + "latitude": 71.701551, + "longitude": 9.903068, + "tags": [] + }, + { + "id": 57, + "isActive": false, + "balance": "$1,844.56", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "Green", + "name": "Kaitlin Conner", + "gender": "female", + "email": "kaitlinconner@chorizon.com", + "phone": "+1 (862) 467-2666", + "address": "501 Knight Court, Joppa, Rhode Island, 274", + "about": "Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n", + "registered": "2019-05-30T06:38:24 -02:00", + "latitude": 15.613464, + "longitude": 171.965629, + "tags": [] + }, + { + "id": 58, + "isActive": true, + "balance": "$2,876.10", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Mamie Fischer", + "gender": "female", + "email": "mamiefischer@chorizon.com", + "phone": "+1 (948) 545-3901", + "address": "599 Hunterfly Place, Haena, Georgia, 6005", + "about": "Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n", + "registered": "2019-05-27T05:07:10 -02:00", + "latitude": 70.915079, + "longitude": -48.813584, + "tags": [ + "bug", + "wontfix", + "wontfix", + "good first issue" + ] + }, + { + "id": 59, + "isActive": true, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ] + }, + { + "id": 60, + "isActive": true, + "balance": "$1,770.93", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "brown", + "name": "Jody Herrera", + "gender": "female", + "email": "jodyherrera@chorizon.com", + "phone": "+1 (890) 583-3222", + "address": "261 Jay Street, Strykersville, Ohio, 9248", + "about": "Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n", + "registered": "2016-05-21T01:00:02 -02:00", + "latitude": -36.846586, + "longitude": 131.156223, + "tags": [] + }, + { + "id": 61, + "isActive": false, + "balance": "$2,813.41", + "picture": "http://placehold.it/32x32", + "age": 37, + "color": "Green", + "name": "Charles Castillo", + "gender": "male", + "email": "charlescastillo@chorizon.com", + "phone": "+1 (934) 467-2108", + "address": "675 Morton Street, Rew, Pennsylvania, 137", + "about": "Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n", + "registered": "2019-06-10T02:54:22 -02:00", + "latitude": -16.423202, + "longitude": -146.293752, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 62, + "isActive": true, + "balance": "$3,341.35", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "blue", + "name": "Estelle Ramirez", + "gender": "female", + "email": "estelleramirez@chorizon.com", + "phone": "+1 (816) 459-2073", + "address": "636 Nolans Lane, Camptown, California, 7794", + "about": "Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n", + "registered": "2015-02-14T01:05:50 -01:00", + "latitude": -46.591249, + "longitude": -83.385587, + "tags": [ + "good first issue", + "bug" + ] + }, + { + "id": 63, + "isActive": true, + "balance": "$2,478.30", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "blue", + "name": "Knowles Hebert", + "gender": "male", + "email": "knowleshebert@chorizon.com", + "phone": "+1 (819) 409-2308", + "address": "361 Kathleen Court, Gratton, Connecticut, 7254", + "about": "Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n", + "registered": "2016-03-08T08:34:52 -01:00", + "latitude": 71.042482, + "longitude": 152.460406, + "tags": [ + "good first issue", + "wontfix" + ] + }, + { + "id": 64, + "isActive": false, + "balance": "$2,559.09", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Thelma Mckenzie", + "gender": "female", + "email": "thelmamckenzie@chorizon.com", + "phone": "+1 (941) 596-2777", + "address": "202 Leonard Street, Riverton, Illinois, 8577", + "about": "Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n", + "registered": "2020-04-14T12:43:06 -02:00", + "latitude": 16.026129, + "longitude": 105.464476, + "tags": [] + }, + { + "id": 65, + "isActive": true, + "balance": "$1,025.08", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Carole Rowland", + "gender": "female", + "email": "carolerowland@chorizon.com", + "phone": "+1 (862) 558-3448", + "address": "941 Melba Court, Bluetown, Florida, 9555", + "about": "Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n", + "registered": "2014-12-01T05:55:35 -01:00", + "latitude": -0.191998, + "longitude": 43.389652, + "tags": [ + "wontfix" + ] + }, + { + "id": 66, + "isActive": true, + "balance": "$1,061.49", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "brown", + "name": "Higgins Aguilar", + "gender": "male", + "email": "higginsaguilar@chorizon.com", + "phone": "+1 (911) 540-3791", + "address": "132 Sackman Street, Layhill, Guam, 8729", + "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", + "registered": "2015-04-05T02:10:07 -02:00", + "latitude": 74.702813, + "longitude": 151.314972, + "tags": [ + "bug" + ] + }, + { + "id": 67, + "isActive": true, + "balance": "$3,510.14", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Ilene Gillespie", + "gender": "female", + "email": "ilenegillespie@chorizon.com", + "phone": "+1 (937) 575-2676", + "address": "835 Lake Street, Naomi, Alabama, 4131", + "about": "Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n", + "registered": "2015-06-28T09:41:45 -02:00", + "latitude": 71.573342, + "longitude": -95.295989, + "tags": [ + "wontfix", + "wontfix" + ] + }, + { + "id": 68, + "isActive": false, + "balance": "$1,539.98", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Angelina Dyer", + "gender": "female", + "email": "angelinadyer@chorizon.com", + "phone": "+1 (948) 574-3949", + "address": "575 Division Place, Gorham, Louisiana, 3458", + "about": "Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n", + "registered": "2014-07-08T06:34:36 -02:00", + "latitude": -85.649593, + "longitude": 66.126018, + "tags": [ + "good first issue" + ] + }, + { + "id": 69, + "isActive": true, + "balance": "$3,367.69", + "picture": "http://placehold.it/32x32", + "age": 30, + "color": "brown", + "name": "Marks Burt", + "gender": "male", + "email": "marksburt@chorizon.com", + "phone": "+1 (895) 497-3138", + "address": "819 Village Road, Wadsworth, Delaware, 6099", + "about": "Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n", + "registered": "2014-08-31T06:12:18 -02:00", + "latitude": 26.854112, + "longitude": -143.313948, + "tags": [ + "good first issue" + ] + }, + { + "id": 70, + "isActive": false, + "balance": "$3,755.72", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "blue", + "name": "Glass Perkins", + "gender": "male", + "email": "glassperkins@chorizon.com", + "phone": "+1 (923) 486-3725", + "address": "899 Roosevelt Court, Belleview, Idaho, 1737", + "about": "Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n", + "registered": "2015-05-22T05:44:33 -02:00", + "latitude": 54.27147, + "longitude": -65.065604, + "tags": [ + "wontfix" + ] + }, + { + "id": 71, + "isActive": true, + "balance": "$3,381.63", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Candace Sawyer", + "gender": "female", + "email": "candacesawyer@chorizon.com", + "phone": "+1 (830) 404-2636", + "address": "334 Arkansas Drive, Bordelonville, Tennessee, 8449", + "about": "Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n", + "registered": "2014-04-04T08:45:00 -02:00", + "latitude": 6.484262, + "longitude": -37.054928, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 72, + "isActive": true, + "balance": "$1,640.98", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Hendricks Martinez", + "gender": "male", + "email": "hendricksmartinez@chorizon.com", + "phone": "+1 (857) 566-3245", + "address": "636 Agate Court, Newry, Utah, 3304", + "about": "Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n", + "registered": "2018-06-15T10:36:11 -02:00", + "latitude": 86.746034, + "longitude": 10.347893, + "tags": [ + "new issue" + ] + }, + { + "id": 73, + "isActive": false, + "balance": "$1,239.74", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "blue", + "name": "Eleanor Shepherd", + "gender": "female", + "email": "eleanorshepherd@chorizon.com", + "phone": "+1 (894) 567-2617", + "address": "670 Lafayette Walk, Darlington, Palau, 8803", + "about": "Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n", + "registered": "2020-02-29T12:15:28 -01:00", + "latitude": 35.749621, + "longitude": -94.40842, + "tags": [ + "good first issue", + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 74, + "isActive": true, + "balance": "$1,180.90", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Stark Wong", + "gender": "male", + "email": "starkwong@chorizon.com", + "phone": "+1 (805) 575-3055", + "address": "522 Bond Street, Bawcomville, Wisconsin, 324", + "about": "Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n", + "registered": "2020-01-25T10:47:48 -01:00", + "latitude": -80.452139, + "longitude": 160.72546, + "tags": [ + "wontfix" + ] + }, + { + "id": 75, + "isActive": false, + "balance": "$1,913.42", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Emma Jacobs", + "gender": "female", + "email": "emmajacobs@chorizon.com", + "phone": "+1 (899) 554-3847", + "address": "173 Tapscott Street, Esmont, Maine, 7450", + "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", + "registered": "2019-03-29T06:24:13 -01:00", + "latitude": -35.53722, + "longitude": 155.703874, + "tags": [] + }, + { + "id": 76, + "isActive": false, + "balance": "$1,274.29", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "Green", + "name": "Clarice Gardner", + "gender": "female", + "email": "claricegardner@chorizon.com", + "phone": "+1 (810) 407-3258", + "address": "894 Brooklyn Road, Utting, New Hampshire, 6404", + "about": "Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n", + "registered": "2014-10-20T10:13:32 -02:00", + "latitude": 17.11935, + "longitude": 65.38197, + "tags": [ + "new issue", + "wontfix" + ] + } +] From 5ecf514d28a2e9258debee05046941e1f9dca5cc Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 13:46:49 +0100 Subject: [PATCH 150/522] restructure project --- .gitignore | 8 +- Cargo.lock | 496 +-- Cargo.toml | 88 +- Dockerfile | 29 + meilisearch-error/Cargo.toml | 8 + meilisearch-error/src/lib.rs | 185 + meilisearch-http/Cargo.lock | 3804 +++++++++++++++++ meilisearch-http/Cargo.toml | 82 + build.rs => meilisearch-http/build.rs | 0 .../public}/bulma.min.css | 0 .../public}/interface.html | 0 {src => meilisearch-http/src}/analytics.rs | 0 {src => meilisearch-http/src}/data/mod.rs | 0 {src => meilisearch-http/src}/data/search.rs | 0 {src => meilisearch-http/src}/data/updates.rs | 0 {src => meilisearch-http/src}/dump.rs | 0 {src => meilisearch-http/src}/error.rs | 0 .../src}/helpers/authentication.rs | 0 .../src}/helpers/compression.rs | 0 {src => meilisearch-http/src}/helpers/mod.rs | 0 .../src}/helpers/normalize_path.rs | 0 {src => meilisearch-http/src}/index/mod.rs | 0 {src => meilisearch-http/src}/index/search.rs | 0 .../src}/index/updates.rs | 0 .../actor_index_controller/mod.rs | 0 .../src}/index_controller/index_actor.rs | 0 .../local_index_controller/index_store.rs | 0 .../local_index_controller/mod.rs | 0 .../src}/index_controller/mod.rs | 0 .../src}/index_controller/update_actor.rs | 0 .../src}/index_controller/update_handler.rs | 0 .../src}/index_controller/update_store.rs | 0 .../src}/index_controller/updates.rs | 0 .../src}/index_controller/uuid_resolver.rs | 0 {src => meilisearch-http/src}/lib.rs | 0 {src => meilisearch-http/src}/main.rs | 0 {src => meilisearch-http/src}/option.rs | 0 .../src}/routes/document.rs | 0 {src => meilisearch-http/src}/routes/dump.rs | 0 .../src}/routes/health.rs | 0 {src => meilisearch-http/src}/routes/index.rs | 0 {src => meilisearch-http/src}/routes/key.rs | 0 {src => meilisearch-http/src}/routes/mod.rs | 0 .../src}/routes/search.rs | 0 .../settings/attributes_for_faceting.rs | 0 .../routes/settings/displayed_attributes.rs | 0 .../routes/settings/distinct_attributes.rs | 0 .../src}/routes/settings/mod.rs | 0 .../src}/routes/settings/ranking_rules.rs | 0 .../routes/settings/searchable_attributes.rs | 0 .../src}/routes/settings/stop_words.rs | 0 .../src}/routes/settings/synonyms.rs | 0 {src => meilisearch-http/src}/routes/stats.rs | 0 .../src}/routes/stop_words.rs | 0 .../src}/routes/synonym.rs | 0 {src => meilisearch-http/src}/snapshot.rs | 0 .../tests}/assets/test_set.json | 0 .../tests}/common/index.rs | 0 .../tests}/common/mod.rs | 0 .../tests}/common/server.rs | 0 .../tests}/common/service.rs | 0 .../tests}/documents/add_documents.rs | 0 .../tests}/documents/delete_documents.rs | 0 .../tests}/documents/get_documents.rs | 0 .../tests}/documents/mod.rs | 0 .../tests}/index/create_index.rs | 0 .../tests}/index/delete_index.rs | 0 .../tests}/index/get_index.rs | 0 .../tests}/index/mod.rs | 0 .../tests}/index/update_index.rs | 0 .../tests}/integration.rs | 0 .../tests}/search/mod.rs | 0 .../tests}/settings/get_settings.rs | 0 .../tests}/settings/mod.rs | 0 .../tests}/updates/mod.rs | 0 75 files changed, 4377 insertions(+), 323 deletions(-) create mode 100644 Dockerfile create mode 100644 meilisearch-error/Cargo.toml create mode 100644 meilisearch-error/src/lib.rs create mode 100644 meilisearch-http/Cargo.lock create mode 100644 meilisearch-http/Cargo.toml rename build.rs => meilisearch-http/build.rs (100%) rename {public => meilisearch-http/public}/bulma.min.css (100%) rename {public => meilisearch-http/public}/interface.html (100%) rename {src => meilisearch-http/src}/analytics.rs (100%) rename {src => meilisearch-http/src}/data/mod.rs (100%) rename {src => meilisearch-http/src}/data/search.rs (100%) rename {src => meilisearch-http/src}/data/updates.rs (100%) rename {src => meilisearch-http/src}/dump.rs (100%) rename {src => meilisearch-http/src}/error.rs (100%) rename {src => meilisearch-http/src}/helpers/authentication.rs (100%) rename {src => meilisearch-http/src}/helpers/compression.rs (100%) rename {src => meilisearch-http/src}/helpers/mod.rs (100%) rename {src => meilisearch-http/src}/helpers/normalize_path.rs (100%) rename {src => meilisearch-http/src}/index/mod.rs (100%) rename {src => meilisearch-http/src}/index/search.rs (100%) rename {src => meilisearch-http/src}/index/updates.rs (100%) rename {src => meilisearch-http/src}/index_controller/actor_index_controller/mod.rs (100%) rename {src => meilisearch-http/src}/index_controller/index_actor.rs (100%) rename {src => meilisearch-http/src}/index_controller/local_index_controller/index_store.rs (100%) rename {src => meilisearch-http/src}/index_controller/local_index_controller/mod.rs (100%) rename {src => meilisearch-http/src}/index_controller/mod.rs (100%) rename {src => meilisearch-http/src}/index_controller/update_actor.rs (100%) rename {src => meilisearch-http/src}/index_controller/update_handler.rs (100%) rename {src => meilisearch-http/src}/index_controller/update_store.rs (100%) rename {src => meilisearch-http/src}/index_controller/updates.rs (100%) rename {src => meilisearch-http/src}/index_controller/uuid_resolver.rs (100%) rename {src => meilisearch-http/src}/lib.rs (100%) rename {src => meilisearch-http/src}/main.rs (100%) rename {src => meilisearch-http/src}/option.rs (100%) rename {src => meilisearch-http/src}/routes/document.rs (100%) rename {src => meilisearch-http/src}/routes/dump.rs (100%) rename {src => meilisearch-http/src}/routes/health.rs (100%) rename {src => meilisearch-http/src}/routes/index.rs (100%) rename {src => meilisearch-http/src}/routes/key.rs (100%) rename {src => meilisearch-http/src}/routes/mod.rs (100%) rename {src => meilisearch-http/src}/routes/search.rs (100%) rename {src => meilisearch-http/src}/routes/settings/attributes_for_faceting.rs (100%) rename {src => meilisearch-http/src}/routes/settings/displayed_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/distinct_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/mod.rs (100%) rename {src => meilisearch-http/src}/routes/settings/ranking_rules.rs (100%) rename {src => meilisearch-http/src}/routes/settings/searchable_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/stop_words.rs (100%) rename {src => meilisearch-http/src}/routes/settings/synonyms.rs (100%) rename {src => meilisearch-http/src}/routes/stats.rs (100%) rename {src => meilisearch-http/src}/routes/stop_words.rs (100%) rename {src => meilisearch-http/src}/routes/synonym.rs (100%) rename {src => meilisearch-http/src}/snapshot.rs (100%) rename {tests => meilisearch-http/tests}/assets/test_set.json (100%) rename {tests => meilisearch-http/tests}/common/index.rs (100%) rename {tests => meilisearch-http/tests}/common/mod.rs (100%) rename {tests => meilisearch-http/tests}/common/server.rs (100%) rename {tests => meilisearch-http/tests}/common/service.rs (100%) rename {tests => meilisearch-http/tests}/documents/add_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/delete_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/get_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/mod.rs (100%) rename {tests => meilisearch-http/tests}/index/create_index.rs (100%) rename {tests => meilisearch-http/tests}/index/delete_index.rs (100%) rename {tests => meilisearch-http/tests}/index/get_index.rs (100%) rename {tests => meilisearch-http/tests}/index/mod.rs (100%) rename {tests => meilisearch-http/tests}/index/update_index.rs (100%) rename {tests => meilisearch-http/tests}/integration.rs (100%) rename {tests => meilisearch-http/tests}/search/mod.rs (100%) rename {tests => meilisearch-http/tests}/settings/get_settings.rs (100%) rename {tests => meilisearch-http/tests}/settings/mod.rs (100%) rename {tests => meilisearch-http/tests}/updates/mod.rs (100%) diff --git a/.gitignore b/.gitignore index 1d71f78ca..e1f56a99c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ -# the milli project is a library /target +meilisearch-core/target +**/*.csv +**/*.json_lines +**/*.rs.bk +/*.mdb +/query-history.txt +/data.ms diff --git a/Cargo.lock b/Cargo.lock index 0bdc739d5..7f8724ca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "actix-codec" version = "0.3.0" @@ -14,7 +12,7 @@ dependencies = [ "futures-sink", "log", "pin-project 0.4.27", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-util 0.3.1", ] @@ -29,9 +27,9 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.0", - "tokio 1.2.0", - "tokio-util 0.6.3", + "pin-project-lite 0.2.6", + "tokio 1.3.0", + "tokio-util 0.6.4", ] [[package]] @@ -99,15 +97,15 @@ dependencies = [ "log", "mime", "percent-encoding", - "pin-project 1.0.2", + "pin-project 1.0.5", "rand 0.7.3", "regex", "serde", "serde_json", "serde_urlencoded", - "sha-1 0.9.2", + "sha-1 0.9.4", "slab", - "time 0.2.23", + "time 0.2.25", ] [[package]] @@ -121,7 +119,7 @@ dependencies = [ "actix-service 2.0.0-beta.4", "actix-tls", "actix-utils 3.0.0-beta.2", - "ahash 0.7.0", + "ahash 0.7.2", "base64 0.13.0", "bitflags", "brotli2", @@ -143,16 +141,16 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "pin-project 1.0.2", + "pin-project 1.0.5", "rand 0.8.3", "regex", "serde", "serde_json", "serde_urlencoded", - "sha-1 0.9.2", + "sha-1 0.9.4", "smallvec", - "time 0.2.23", - "tokio 1.2.0", + "time 0.2.25", + "tokio 1.3.0", ] [[package]] @@ -200,7 +198,7 @@ dependencies = [ "futures-channel", "futures-util", "smallvec", - "tokio 0.2.24", + "tokio 0.2.25", ] [[package]] @@ -211,7 +209,7 @@ checksum = "0b4e57bc1a3915e71526d128baf4323700bd1580bc676239e2298a4c5b001f18" dependencies = [ "actix-macros 0.2.0", "futures-core", - "tokio 1.2.0", + "tokio 1.3.0", ] [[package]] @@ -229,7 +227,7 @@ dependencies = [ "mio 0.7.9", "num_cpus", "slab", - "tokio 1.2.0", + "tokio 1.3.0", ] [[package]] @@ -249,7 +247,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca9756f4d32984ac454ae3155a276f6be69b424197bd3f0ca3c87cde72f41d63" dependencies = [ "futures-core", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", ] [[package]] @@ -282,7 +280,7 @@ dependencies = [ "http", "log", "tokio-rustls 0.22.0", - "tokio-util 0.6.3", + "tokio-util 0.6.4", "webpki-roots 0.21.0", ] @@ -318,7 +316,7 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", ] [[package]] @@ -337,7 +335,7 @@ dependencies = [ "actix-tls", "actix-utils 3.0.0-beta.2", "actix-web-codegen", - "ahash 0.7.0", + "ahash 0.7.2", "awc", "bytes 1.0.1", "derive_more", @@ -347,7 +345,7 @@ dependencies = [ "futures-util", "log", "mime", - "pin-project 1.0.2", + "pin-project 1.0.5", "regex", "rustls 0.19.0", "serde", @@ -355,7 +353,7 @@ dependencies = [ "serde_urlencoded", "smallvec", "socket2", - "time 0.2.23", + "time 0.2.25", "url", ] @@ -372,18 +370,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" dependencies = [ "gimli", ] [[package]] name = "adler" -version = "0.2.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" @@ -393,9 +391,9 @@ checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" [[package]] name = "ahash" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa60d2eadd8b12a996add391db32bd1153eac697ba4869660c0016353611426" +checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" dependencies = [ "getrandom 0.2.2", "once_cell", @@ -437,15 +435,15 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a" +checksum = "b72c1f1154e234325b50864a349b9c8e56939e266a4c307c0f159812df2f9537" dependencies = [ "flate2", "futures-core", "memchr", - "pin-project-lite 0.1.11", - "tokio 0.2.24", + "pin-project-lite 0.2.6", + "tokio 0.2.25", ] [[package]] @@ -471,9 +469,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.42" +version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ "proc-macro2", "quote", @@ -516,7 +514,7 @@ dependencies = [ "log", "mime", "percent-encoding", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "rand 0.8.3", "rustls 0.19.0", "serde", @@ -526,9 +524,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.55" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -590,7 +588,7 @@ dependencies = [ "block-padding", "byte-tools", "byteorder", - "generic-array 0.12.3", + "generic-array 0.12.4", ] [[package]] @@ -645,9 +643,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.4.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" [[package]] name = "byte-tools" @@ -672,9 +670,9 @@ checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" [[package]] name = "byteorder" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" @@ -705,9 +703,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" dependencies = [ "jobserver", ] @@ -773,18 +771,18 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" [[package]] name = "cookie" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ "percent-encoding", - "time 0.2.23", + "time 0.2.25", "version_check", ] @@ -822,7 +820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", ] [[package]] @@ -833,18 +831,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", ] [[package]] name = "crossbeam-epoch" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" dependencies = [ "cfg-if 1.0.0", - "const_fn", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", "lazy_static", "memoffset", "scopeguard", @@ -871,9 +868,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -882,9 +879,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" dependencies = [ "bstr", "csv-core", @@ -945,7 +942,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" dependencies = [ - "generic-array 0.12.3", + "generic-array 0.12.4", ] [[package]] @@ -971,9 +968,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.26" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" dependencies = [ "cfg-if 1.0.0", ] @@ -1005,12 +1002,12 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" dependencies = [ "atty", - "humantime 2.0.1", + "humantime 2.1.0", "log", "regex", "termcolor", @@ -1046,13 +1043,13 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "filetime" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "winapi 0.3.9", ] @@ -1076,9 +1073,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", "percent-encoding", @@ -1120,9 +1117,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" dependencies = [ "futures-channel", "futures-core", @@ -1135,9 +1132,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" dependencies = [ "futures-core", "futures-sink", @@ -1145,15 +1142,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" +checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" [[package]] name = "futures-executor" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" dependencies = [ "futures-core", "futures-task", @@ -1162,15 +1159,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" +checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" [[package]] name = "futures-macro" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -1180,24 +1177,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" +checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" [[package]] name = "futures-task" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" -dependencies = [ - "once_cell", -] +checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" [[package]] name = "futures-util" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" dependencies = [ "futures-channel", "futures-core", @@ -1206,7 +1200,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.2", + "pin-project-lite 0.2.6", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -1224,9 +1218,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" dependencies = [ "typenum", ] @@ -1243,11 +1237,11 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1303,7 +1297,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-util 0.3.1", "tracing", "tracing-futures", @@ -1323,8 +1317,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.2.0", - "tokio-util 0.6.3", + "tokio 1.3.0", + "tokio-util 0.6.4", "tracing", ] @@ -1393,9 +1387,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] @@ -1434,9 +1428,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.4" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" [[package]] name = "httpdate" @@ -1461,15 +1455,15 @@ dependencies = [ [[package]] name = "humantime" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.13.9" +version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" dependencies = [ "bytes 0.5.6", "futures-channel", @@ -1481,9 +1475,9 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.2", + "pin-project 1.0.5", "socket2", - "tokio 0.2.24", + "tokio 0.2.25", "tower-service", "tracing", "want", @@ -1500,16 +1494,16 @@ dependencies = [ "hyper", "log", "rustls 0.18.1", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-rustls 0.14.1", "webpki", ] [[package]] name = "idna" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" dependencies = [ "matches", "unicode-bidi", @@ -1532,9 +1526,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", "hashbrown 0.9.1", @@ -1597,9 +1591,9 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jemalloc-sys" @@ -1648,9 +1642,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.46" +version = "0.3.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" dependencies = [ "wasm-bindgen", ] @@ -1688,9 +1682,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.86" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" +checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a" [[package]] name = "linked-hash-map" @@ -1787,7 +1781,7 @@ dependencies = [ "crossbeam-channel", "dashmap", "either", - "env_logger 0.8.2", + "env_logger 0.8.3", "flate2", "fst", "futures", @@ -1822,7 +1816,7 @@ dependencies = [ "tempdir", "tempfile", "thiserror", - "tokio 1.2.0", + "tokio 1.3.0", "uuid", "vergen", ] @@ -1830,7 +1824,7 @@ dependencies = [ [[package]] name = "meilisearch-tokenizer" version = "0.1.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#147b6154b1b34cb8f5da2df6a416b7da191bc850" +source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#31ba3ff4a15501f12b7d37ac64ddce7c35a9757c" dependencies = [ "character_converter", "cow-utils", @@ -1926,9 +1920,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", "autocfg", @@ -2062,9 +2056,9 @@ dependencies = [ [[package]] name = "object" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" [[package]] name = "obkv" @@ -2074,9 +2068,9 @@ checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" [[package]] name = "once_cell" -version = "1.5.2" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" [[package]] name = "opaque-debug" @@ -2122,14 +2116,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "smallvec", "winapi 0.3.9", ] @@ -2240,11 +2234,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" dependencies = [ - "pin-project-internal 1.0.2", + "pin-project-internal 1.0.5", ] [[package]] @@ -2260,9 +2254,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ "proc-macro2", "quote", @@ -2271,15 +2265,15 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" +checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" [[package]] name = "pin-utils" @@ -2331,9 +2325,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro-nested" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" @@ -2352,9 +2346,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] @@ -2378,7 +2372,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -2439,7 +2433,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", ] [[package]] @@ -2507,7 +2501,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", "lazy_static", "num_cpus", ] @@ -2521,12 +2515,6 @@ dependencies = [ "rand_core 0.3.1", ] -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.2.5" @@ -2594,12 +2582,12 @@ dependencies = [ "mime", "mime_guess", "percent-encoding", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-rustls 0.14.1", "url", "wasm-bindgen", @@ -2627,9 +2615,9 @@ checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" [[package]] name = "ring" -version = "0.16.19" +version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ "cc", "libc", @@ -2663,7 +2651,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", ] [[package]] @@ -2720,7 +2717,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", ] [[package]] @@ -2729,6 +2735,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "sentry" version = "0.18.1" @@ -2747,7 +2762,7 @@ dependencies = [ "rand 0.7.3", "regex", "reqwest", - "rustc_version", + "rustc_version 0.2.3", "sentry-types", "uname", "url", @@ -2770,18 +2785,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.123" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.123" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ "proc-macro2", "quote", @@ -2802,9 +2817,9 @@ dependencies = [ [[package]] name = "serde_url_params" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +checksum = "2c43307d0640738af32fe8d01e47119bc0fc8a686be470a44a586caff76dfb34" dependencies = [ "serde", "url", @@ -2836,9 +2851,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -2855,9 +2870,9 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -2921,9 +2936,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98d3306e84bf86710d6cd8b4c9c3b721d5454cc91a603180f8f8cd06cfd317b4" +checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a" [[package]] name = "socket2" @@ -2944,9 +2959,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.13" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" +checksum = "a2beb4d1860a61f571530b3f855a1b538d0200f7871c63331ecd6f17b1f014f8" dependencies = [ "version_check", ] @@ -2958,7 +2973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -3032,9 +3047,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717" dependencies = [ "proc-macro2", "quote", @@ -3064,13 +3079,12 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.30" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +checksum = "c0bcfbd6a598361fda270d82469fff3d65089dc33e175c9a131f7b4cd395f228" dependencies = [ "filetime", "libc", - "redox_syscall 0.1.57", "xattr", ] @@ -3093,7 +3107,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall 0.2.5", + "redox_syscall", "remove_dir_all", "winapi 0.3.9", ] @@ -3138,11 +3152,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.0.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] @@ -3167,9 +3181,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +checksum = "1195b046942c221454c2539395f85413b33383a067449d78aab2b7b052a142f7" dependencies = [ "const_fn", "libc", @@ -3205,9 +3219,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" dependencies = [ "tinyvec_macros", ] @@ -3220,9 +3234,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" dependencies = [ "bytes 0.5.6", "fnv", @@ -3234,7 +3248,7 @@ dependencies = [ "mio 0.6.23", "mio-uds", "num_cpus", - "pin-project-lite 0.1.11", + "pin-project-lite 0.1.12", "signal-hook-registry", "slab", "winapi 0.3.9", @@ -3242,9 +3256,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda" dependencies = [ "autocfg", "bytes 1.0.1", @@ -3254,7 +3268,7 @@ dependencies = [ "num_cpus", "once_cell", "parking_lot", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "signal-hook-registry", "tokio-macros", "winapi 0.3.9", @@ -3279,7 +3293,7 @@ checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", "rustls 0.18.1", - "tokio 0.2.24", + "tokio 0.2.25", "webpki", ] @@ -3290,7 +3304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ "rustls 0.19.0", - "tokio 1.2.0", + "tokio 1.3.0", "webpki", ] @@ -3304,39 +3318,39 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.1.11", - "tokio 0.2.24", + "pin-project-lite 0.1.12", + "tokio 0.2.25", ] [[package]] name = "tokio-util" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +checksum = "ec31e5cc6b46e653cf57762f36f71d5e6386391d88a72fd6db4508f8f676fb29" dependencies = [ "bytes 1.0.1", "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.0", - "tokio 1.2.0", + "pin-project-lite 0.2.6", + "tokio 1.3.0", ] [[package]] name = "tower-service" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.22" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "tracing-core", ] @@ -3351,22 +3365,22 @@ dependencies = [ [[package]] name = "tracing-futures" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 0.4.27", + "pin-project 1.0.5", "tracing", ] [[package]] name = "trust-dns-proto" -version = "0.19.6" +version = "0.19.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" +checksum = "1cad71a0c0d68ab9941d2fb6e82f8fb2e86d9945b94e1661dd0aaea2b88215a9" dependencies = [ "async-trait", - "backtrace", + "cfg-if 1.0.0", "enum-as-inner", "futures", "idna", @@ -3375,17 +3389,16 @@ dependencies = [ "rand 0.7.3", "smallvec", "thiserror", - "tokio 0.2.24", + "tokio 0.2.25", "url", ] [[package]] name = "trust-dns-resolver" -version = "0.19.6" +version = "0.19.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" +checksum = "710f593b371175db53a26d0b38ed2978fafb9e9e8d3868b1acd753ea18df0ceb" dependencies = [ - "backtrace", "cfg-if 0.1.10", "futures", "ipconfig", @@ -3395,7 +3408,7 @@ dependencies = [ "resolv-conf", "smallvec", "thiserror", - "tokio 0.2.24", + "tokio 0.2.25", "trust-dns-proto", ] @@ -3446,9 +3459,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" dependencies = [ "tinyvec", ] @@ -3479,9 +3492,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" dependencies = [ "form_urlencoded", "idna", @@ -3514,12 +3527,13 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vergen" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" dependencies = [ "bitflags", "chrono", + "rustc_version 0.3.3", ] [[package]] @@ -3552,9 +3566,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" dependencies = [ "cfg-if 1.0.0", "serde", @@ -3564,9 +3578,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" dependencies = [ "bumpalo", "lazy_static", @@ -3579,9 +3593,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.19" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +checksum = "8e67a5806118af01f0d9045915676b22aaebecf4178ae7021bc171dab0b897ab" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3591,9 +3605,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3601,9 +3615,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" dependencies = [ "proc-macro2", "quote", @@ -3614,15 +3628,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" +checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" [[package]] name = "web-sys" -version = "0.3.46" +version = "0.3.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 792ebd763..a1dca038e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,82 +1,8 @@ -[package] -authors = ["Quentin de Quelen ", "Clément Renault "] -description = "MeiliSearch HTTP server" -edition = "2018" -license = "MIT" -name = "meilisearch-http" -version = "0.17.0" -[[bin]] -name = "meilisearch" -path = "src/main.rs" +[workspace] +members = [ + "meilisearch-http", + "meilisearch-error", +] -[build-dependencies] -vergen = "3.1.0" - -[dependencies] -actix-cors = { path = "../actix-extras/actix-cors" } -actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } -actix-service = "2.0.0-beta.4" -actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } -#actix-web = { version = "3", features = ["rustls"] } -anyhow = "1.0.36" -async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } -byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } -bytes = "0.6.0" -chrono = { version = "0.4.19", features = ["serde"] } -crossbeam-channel = "0.5.0" -env_logger = "0.8.2" -flate2 = "1.0.19" -fst = "0.4.5" -futures = "0.3.7" -futures-util = "0.3.8" -grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = "0.10.6" -http = "0.2.1" -indexmap = { version = "1.3.2", features = ["serde-1"] } -log = "0.4.8" -main_error = "0.1.0" -meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } -memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } -mime = "0.3.16" -once_cell = "1.5.2" -rand = "0.7.3" -rayon = "1.5.0" -regex = "1.4.2" -rustls = "0.19" -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0.59", features = ["preserve_order"] } -sha2 = "0.9.1" -siphasher = "0.3.2" -slice-group-by = "0.2.6" -structopt = "0.3.20" -tar = "0.4.29" -tempfile = "3.1.0" -tokio = { version = "1", features = ["full"] } -dashmap = "4.0.2" -uuid = "0.8.2" -itertools = "0.10.0" -either = "1.6.1" -async-trait = "0.1.42" -thiserror = "1.0.24" -async-stream = "0.3.0" - -[dependencies.sentry] -default-features = false -features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] -optional = true -version = "0.18.1" - - -[dev-dependencies] -serde_url_params = "0.2.0" -tempdir = "0.3.7" -assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } -actix-rt = "2.1.0" - -[features] -default = ["sentry"] - -[target.'cfg(unix)'.dependencies] -jemallocator = "0.3.2" +[profile.release] +debug = true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..9898d02db --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Compile +FROM alpine:3.10 AS compiler + +RUN apk update --quiet +RUN apk add curl +RUN apk add build-base + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +WORKDIR /meilisearch + +COPY . . + +ENV RUSTFLAGS="-C target-feature=-crt-static" + +RUN $HOME/.cargo/bin/cargo build --release + +# Run +FROM alpine:3.10 + +RUN apk add -q --no-cache libgcc tini + +COPY --from=compiler /meilisearch/target/release/meilisearch . + +ENV MEILI_HTTP_ADDR 0.0.0.0:7700 +EXPOSE 7700/tcp + +ENTRYPOINT ["tini", "--"] +CMD ./meilisearch diff --git a/meilisearch-error/Cargo.toml b/meilisearch-error/Cargo.toml new file mode 100644 index 000000000..d5a474ea5 --- /dev/null +++ b/meilisearch-error/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "meilisearch-error" +version = "0.19.0" +authors = ["marin "] +edition = "2018" + +[dependencies] +actix-http = "2.2.0" diff --git a/meilisearch-error/src/lib.rs b/meilisearch-error/src/lib.rs new file mode 100644 index 000000000..d0e00e9be --- /dev/null +++ b/meilisearch-error/src/lib.rs @@ -0,0 +1,185 @@ +use std::fmt; + +use actix_http::http::StatusCode; + +pub trait ErrorCode: std::error::Error { + fn error_code(&self) -> Code; + + /// returns the HTTP status code ascociated with the error + fn http_status(&self) -> StatusCode { + self.error_code().http() + } + + /// returns the doc url ascociated with the error + fn error_url(&self) -> String { + self.error_code().url() + } + + /// returns error name, used as error code + fn error_name(&self) -> String { + self.error_code().name() + } + + /// return the error type + fn error_type(&self) -> String { + self.error_code().type_() + } +} + +#[allow(clippy::enum_variant_names)] +enum ErrorType { + InternalError, + InvalidRequestError, + AuthenticationError, +} + +impl fmt::Display for ErrorType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use ErrorType::*; + + match self { + InternalError => write!(f, "internal_error"), + InvalidRequestError => write!(f, "invalid_request_error"), + AuthenticationError => write!(f, "authentication_error"), + } + } +} + +pub enum Code { + // index related error + CreateIndex, + IndexAlreadyExists, + IndexNotFound, + InvalidIndexUid, + OpenIndex, + + // invalid state error + InvalidState, + MissingPrimaryKey, + PrimaryKeyAlreadyPresent, + + MaxFieldsLimitExceeded, + MissingDocumentId, + + Facet, + Filter, + + BadParameter, + BadRequest, + DocumentNotFound, + Internal, + InvalidToken, + MissingAuthorizationHeader, + NotFound, + PayloadTooLarge, + RetrieveDocument, + SearchDocuments, + UnsupportedMediaType, + + DumpAlreadyInProgress, + DumpProcessFailed, +} + +impl Code { + + /// ascociate a `Code` variant to the actual ErrCode + fn err_code(&self) -> ErrCode { + use Code::*; + + match self { + // index related errors + // create index is thrown on internal error while creating an index. + CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST), + IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST), + // thrown when requesting an unexisting index + IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND), + InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST), + OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR), + + // invalid state error + InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR), + // thrown when no primary key has been set + MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST), + // error thrown when trying to set an already existing primary key + PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST), + + // invalid document + MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST), + MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST), + + // error related to facets + Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST), + // error related to filters + Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST), + + BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST), + BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST), + DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND), + Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR), + InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN), + MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED), + NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND), + PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE), + RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST), + SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST), + UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE), + + // error related to dump + DumpAlreadyInProgress => ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT), + DumpProcessFailed => ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR), + } + } + + /// return the HTTP status code ascociated with the `Code` + fn http(&self) -> StatusCode { + self.err_code().status_code + } + + /// return error name, used as error code + fn name(&self) -> String { + self.err_code().error_name.to_string() + } + + /// return the error type + fn type_(&self) -> String { + self.err_code().error_type.to_string() + } + + /// return the doc url ascociated with the error + fn url(&self) -> String { + format!("https://docs.meilisearch.com/errors#{}", self.name()) + } +} + +/// Internal structure providing a convenient way to create error codes +struct ErrCode { + status_code: StatusCode, + error_type: ErrorType, + error_name: &'static str, +} + +impl ErrCode { + fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::AuthenticationError, + } + } + + fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::InternalError, + } + } + + fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::InvalidRequestError, + } + } +} diff --git a/meilisearch-http/Cargo.lock b/meilisearch-http/Cargo.lock new file mode 100644 index 000000000..0bdc739d5 --- /dev/null +++ b/meilisearch-http/Cargo.lock @@ -0,0 +1,3804 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "actix-codec" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" +dependencies = [ + "bitflags", + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project 0.4.27", + "tokio 0.2.24", + "tokio-util 0.3.1", +] + +[[package]] +name = "actix-codec" +version = "0.4.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e" +dependencies = [ + "bitflags", + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", + "tokio 1.2.0", + "tokio-util 0.6.3", +] + +[[package]] +name = "actix-connect" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" +dependencies = [ + "actix-codec 0.3.0", + "actix-rt 1.1.1", + "actix-service 1.0.6", + "actix-utils 2.0.0", + "derive_more", + "either", + "futures-util", + "http", + "log", + "trust-dns-proto", + "trust-dns-resolver", +] + +[[package]] +name = "actix-cors" +version = "0.5.4" +dependencies = [ + "actix-web", + "derive_more", + "futures-util", + "log", + "once_cell", + "tinyvec", +] + +[[package]] +name = "actix-http" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" +dependencies = [ + "actix-codec 0.3.0", + "actix-connect", + "actix-rt 1.1.1", + "actix-service 1.0.6", + "actix-threadpool", + "actix-utils 2.0.0", + "base64 0.13.0", + "bitflags", + "bytes 0.5.6", + "cookie", + "copyless", + "derive_more", + "either", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "h2 0.2.7", + "http", + "httparse", + "indexmap", + "itoa", + "language-tags", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project 1.0.2", + "rand 0.7.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1 0.9.2", + "slab", + "time 0.2.23", +] + +[[package]] +name = "actix-http" +version = "3.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a01f9e0681608afa887d4269a0857ac4226f09ba5ceda25939e8391c9da610a" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-tls", + "actix-utils 3.0.0-beta.2", + "ahash 0.7.0", + "base64 0.13.0", + "bitflags", + "brotli2", + "bytes 1.0.1", + "bytestring", + "cfg-if 1.0.0", + "cookie", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "futures-util", + "h2 0.3.1", + "http", + "httparse", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project 1.0.2", + "rand 0.8.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1 0.9.2", + "smallvec", + "time 0.2.23", + "tokio 1.2.0", +] + +[[package]] +name = "actix-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" +dependencies = [ + "bytestring", + "http", + "log", + "regex", + "serde", +] + +[[package]] +name = "actix-rt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" +dependencies = [ + "actix-macros 0.1.3", + "actix-threadpool", + "copyless", + "futures-channel", + "futures-util", + "smallvec", + "tokio 0.2.24", +] + +[[package]] +name = "actix-rt" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b4e57bc1a3915e71526d128baf4323700bd1580bc676239e2298a4c5b001f18" +dependencies = [ + "actix-macros 0.2.0", + "futures-core", + "tokio 1.2.0", +] + +[[package]] +name = "actix-server" +version = "2.0.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a99198727204a48f82559c18e4b0ba3197b97d5f4576a32bdbef371f3b4599c1" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-utils 3.0.0-beta.2", + "futures-core", + "log", + "mio 0.7.9", + "num_cpus", + "slab", + "tokio 1.2.0", +] + +[[package]] +name = "actix-service" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" +dependencies = [ + "futures-util", + "pin-project 0.4.27", +] + +[[package]] +name = "actix-service" +version = "2.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca9756f4d32984ac454ae3155a276f6be69b424197bd3f0ca3c87cde72f41d63" +dependencies = [ + "futures-core", + "pin-project-lite 0.2.0", +] + +[[package]] +name = "actix-threadpool" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" +dependencies = [ + "derive_more", + "futures-channel", + "lazy_static", + "log", + "num_cpus", + "parking_lot", + "threadpool", +] + +[[package]] +name = "actix-tls" +version = "3.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b1455e3f7a26d40cfc1080b571f41e8165e5a88e937ed579f7a4b3d55b0370" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-utils 3.0.0-beta.2", + "derive_more", + "futures-core", + "http", + "log", + "tokio-rustls 0.22.0", + "tokio-util 0.6.3", + "webpki-roots 0.21.0", +] + +[[package]] +name = "actix-utils" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" +dependencies = [ + "actix-codec 0.3.0", + "actix-rt 1.1.1", + "actix-service 1.0.6", + "bitflags", + "bytes 0.5.6", + "either", + "futures-channel", + "futures-sink", + "futures-util", + "log", + "pin-project 0.4.27", + "slab", +] + +[[package]] +name = "actix-utils" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458795e09a29bc5557604f9ff6f32236fd0ee457d631672e4ec8f6a0103bb292" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", +] + +[[package]] +name = "actix-web" +version = "4.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d95e50c9e32e8456220b5804867de76e97a86ab8c38b51c9edcccc0f0fddca7" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-http 3.0.0-beta.4", + "actix-macros 0.2.0", + "actix-router", + "actix-rt 2.1.0", + "actix-server", + "actix-service 2.0.0-beta.4", + "actix-tls", + "actix-utils 3.0.0-beta.2", + "actix-web-codegen", + "ahash 0.7.0", + "awc", + "bytes 1.0.1", + "derive_more", + "either", + "encoding_rs", + "futures-core", + "futures-util", + "log", + "mime", + "pin-project 1.0.2", + "regex", + "rustls 0.19.0", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time 0.2.23", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "0.5.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "addr2line" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" + +[[package]] +name = "ahash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa60d2eadd8b12a996add391db32bd1153eac697ba4869660c0016353611426" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "anyhow" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" + +[[package]] +name = "assert-json-diff" +version = "1.0.1" +source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-compression" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite 0.1.11", + "tokio 0.2.24", +] + +[[package]] +name = "async-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "awc" +version = "3.0.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09aecd8728f6491a62b27454ea4b36fb7e50faf32928b0369b644e402c651f4e" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-http 3.0.0-beta.4", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "base64 0.13.0", + "bytes 1.0.1", + "cfg-if 1.0.0", + "derive_more", + "futures-core", + "itoa", + "log", + "mime", + "percent-encoding", + "pin-project-lite 0.2.0", + "rand 0.8.3", + "rustls 0.19.0", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "backtrace" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +dependencies = [ + "addr2line", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bincode" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" +dependencies = [ + "byteorder", + "serde", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "bstr" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byte-unit" +version = "4.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c8758c32833faaae35b24a73d332e62d0528e89076ae841c63940e37008b153" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "bytemuck" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "bytestring" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" +dependencies = [ + "bytes 1.0.1", +] + +[[package]] +name = "cc" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cedarwood" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963e82c7b94163808ca3a452608d260b64ba5bc7b5653b4af1af59887899f48d" +dependencies = [ + "smallvec", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "character_converter" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e48477ece09d6a21c033cb604968524a37782532727055d6f6faafac1781e5c" +dependencies = [ + "bincode", +] + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "const_fn" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" + +[[package]] +name = "cookie" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +dependencies = [ + "percent-encoding", + "time 0.2.23", + "version_check", +] + +[[package]] +name = "copyless" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" + +[[package]] +name = "cow-utils" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.1", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils 0.8.1", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils 0.8.1", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "csv" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + +[[package]] +name = "debugid" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba" +dependencies = [ + "serde", + "uuid", +] + +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "deunicode" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +dependencies = [ + "atty", + "humantime 2.0.1", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "filetime" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.1.57", + "winapi 0.3.9", +] + +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + +[[package]] +name = "fst" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" + +[[package]] +name = "futures-executor" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" + +[[package]] +name = "futures-macro" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" + +[[package]] +name = "futures-task" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project 1.0.2", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "grenad" +version = "0.1.0" +source = "git+https://github.com/Kerollmops/grenad.git?rev=3adcb26#3adcb267dcbc590c7da10eb5f887a254865b3dbe" +dependencies = [ + "byteorder", + "flate2", + "log", + "nix", + "snap", + "tempfile", + "zstd", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 0.2.24", + "tokio-util 0.3.1", + "tracing", + "tracing-futures", +] + +[[package]] +name = "h2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.2.0", + "tokio-util 0.6.3", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" +dependencies = [ + "ahash 0.3.8", + "autocfg", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heed" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" +dependencies = [ + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-rkv-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", + "zerocopy", +] + +[[package]] +name = "heed-traits" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" + +[[package]] +name = "heed-types" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" +dependencies = [ + "bincode", + "heed-traits", + "serde", + "serde_json", + "zerocopy", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + +[[package]] +name = "http" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +dependencies = [ + "bytes 1.0.1", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "human_format" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "humantime" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" + +[[package]] +name = "hyper" +version = "0.13.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.2.7", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.2", + "socket2", + "tokio 0.2.24", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" +dependencies = [ + "bytes 0.5.6", + "futures-util", + "hyper", + "log", + "rustls 0.18.1", + "tokio 0.2.24", + "tokio-rustls 0.14.1", + "webpki", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "im" +version = "14.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696059c87b83c5a258817ecd67c3af915e3ed141891fc35a1e79908801cf0ce7" +dependencies = [ + "bitmaps", + "rand_core 0.5.1", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "indexmap" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +dependencies = [ + "autocfg", + "hashbrown 0.9.1", + "serde", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "jemalloc-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", +] + +[[package]] +name = "jieba-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34fbdeee8786790f4a99fa30ff5c5f88aa5183f7583693e3788d17fc8a48f33a" +dependencies = [ + "cedarwood", + "fxhash", + "hashbrown 0.9.1", + "lazy_static", + "phf", + "phf_codegen", + "regex", +] + +[[package]] +name = "jobserver" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "levenshtein_automata" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a" +dependencies = [ + "fst", +] + +[[package]] +name = "libc" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "lmdb-rkv-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b27470ac25167b3afdfb6af8fcd3bc1be67de50ffbdaf4073378cfded6ae24a5" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "main_error" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb63bb1e282e0b6aba0addb1f0e87cb5181ea68142b2dfd21ba108f8e8088a64" + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "meilisearch-error" +version = "0.19.0" +dependencies = [ + "actix-http 2.2.0", +] + +[[package]] +name = "meilisearch-http" +version = "0.17.0" +dependencies = [ + "actix-cors", + "actix-http 3.0.0-beta.4", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-web", + "anyhow", + "assert-json-diff", + "async-compression", + "async-stream", + "async-trait", + "byte-unit", + "bytes 0.6.0", + "chrono", + "crossbeam-channel", + "dashmap", + "either", + "env_logger 0.8.2", + "flate2", + "fst", + "futures", + "futures-util", + "grenad", + "heed", + "http", + "indexmap", + "itertools 0.10.0", + "jemallocator", + "log", + "main_error", + "meilisearch-error", + "meilisearch-tokenizer", + "memmap", + "milli", + "mime", + "once_cell", + "rand 0.7.3", + "rayon", + "regex", + "rustls 0.19.0", + "sentry", + "serde", + "serde_json", + "serde_url_params", + "sha2", + "siphasher", + "slice-group-by", + "structopt", + "tar", + "tempdir", + "tempfile", + "thiserror", + "tokio 1.2.0", + "uuid", + "vergen", +] + +[[package]] +name = "meilisearch-tokenizer" +version = "0.1.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#147b6154b1b34cb8f5da2df6a416b7da191bc850" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + +[[package]] +name = "milli" +version = "0.1.0" +source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" +dependencies = [ + "anyhow", + "bstr", + "byteorder", + "crossbeam-channel", + "csv", + "either", + "flate2", + "fst", + "fxhash", + "grenad", + "heed", + "human_format", + "itertools 0.10.0", + "levenshtein_automata", + "linked-hash-map", + "log", + "meilisearch-tokenizer", + "memmap", + "num-traits", + "obkv", + "once_cell", + "ordered-float", + "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", + "pest_derive", + "rayon", + "regex", + "roaring", + "serde", + "serde_json", + "smallstr", + "smallvec", + "tempfile", + "uuid", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" +dependencies = [ + "libc", + "log", + "miow 0.3.6", + "ntapi", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nix" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" + +[[package]] +name = "obkv" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" + +[[package]] +name = "once_cell" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "ordered-float" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" +dependencies = [ + "num-traits", +] + +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest" +version = "2.1.3" +source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sha-1 0.8.2", +] + +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared", + "rand 0.7.3", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +dependencies = [ + "pin-project-internal 1.0.2", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-project-lite" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.15", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", + "rand_pcg", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.15", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils 0.8.1", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" +dependencies = [ + "base64 0.13.0", + "bytes 0.5.6", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite 0.2.0", + "rustls 0.18.1", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 0.2.24", + "tokio-rustls 0.14.1", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.20.0", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "retain_mut" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" + +[[package]] +name = "ring" +version = "0.16.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "roaring" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6744a4a918e91359ad1d356a91e2e943a86d9fb9ae77f715d617032ea2af88f" +dependencies = [ + "bytemuck", + "byteorder", + "retain_mut", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64 0.12.3", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "sentry" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b01b723fc1b0a0f9394ca1a8451daec6e20206d47f96c3dceea7fd11ec9eec0" +dependencies = [ + "backtrace", + "env_logger 0.7.1", + "failure", + "hostname", + "httpdate", + "im", + "lazy_static", + "libc", + "log", + "rand 0.7.3", + "regex", + "reqwest", + "rustc_version", + "sentry-types", + "uname", + "url", +] + +[[package]] +name = "sentry-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ec406c11c060c8a7d5d67fc6f4beb2888338dcb12b9af409451995f124749d" +dependencies = [ + "chrono", + "debugid", + "failure", + "serde", + "serde_json", + "url", + "uuid", +] + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_url_params" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +dependencies = [ + "serde", + "url", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha-1" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" + +[[package]] +name = "sized-chunks" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59044ea371ad781ff976f7b06480b9f0180e834eda94114f2afb4afc12b7718" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "slice-group-by" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7474f0b646d228360ab62ed974744617bc869d959eac8403bfa3665931a7fb" + +[[package]] +name = "smallstr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e922794d168678729ffc7e07182721a14219c65814e66e91b839a272fe5ae4f" +dependencies = [ + "serde", + "smallvec", +] + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "snap" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98d3306e84bf86710d6cd8b4c9c3b721d5454cc91a603180f8f8cd06cfd317b4" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "standback" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" +dependencies = [ + "version_check", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synchronoise" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tar" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +dependencies = [ + "filetime", + "libc", + "redox_syscall 0.1.57", + "xattr", +] + +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall 0.2.5", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio 0.6.23", + "mio-uds", + "num_cpus", + "pin-project-lite 0.1.11", + "signal-hook-registry", + "slab", + "winapi 0.3.9", +] + +[[package]] +name = "tokio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +dependencies = [ + "autocfg", + "bytes 1.0.1", + "libc", + "memchr", + "mio 0.7.9", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite 0.2.0", + "signal-hook-registry", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" +dependencies = [ + "futures-core", + "rustls 0.18.1", + "tokio 0.2.24", + "webpki", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.0", + "tokio 1.2.0", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.1.11", + "tokio 0.2.24", +] + +[[package]] +name = "tokio-util" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", + "tokio 1.2.0", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite 0.2.0", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "trust-dns-proto" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" +dependencies = [ + "async-trait", + "backtrace", + "enum-as-inner", + "futures", + "idna", + "lazy_static", + "log", + "rand 0.7.3", + "smallvec", + "thiserror", + "tokio 0.2.24", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" +dependencies = [ + "backtrace", + "cfg-if 0.1.10", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio 0.2.24", + "trust-dns-proto", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "uname" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" +dependencies = [ + "libc", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-width" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.2", + "serde", +] + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "vergen" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +dependencies = [ + "bitflags", + "chrono", +] + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "web-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" +dependencies = [ + "webpki", +] + +[[package]] +name = "webpki-roots" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +dependencies = [ + "webpki", +] + +[[package]] +name = "whatlang" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075" +dependencies = [ + "hashbrown 0.7.2", +] + +[[package]] +name = "widestring" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + +[[package]] +name = "zerocopy" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + +[[package]] +name = "zstd" +version = "0.5.4+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "2.0.6+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.4.18+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" +dependencies = [ + "cc", + "glob", + "itertools 0.9.0", + "libc", +] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml new file mode 100644 index 000000000..0f3d3cb38 --- /dev/null +++ b/meilisearch-http/Cargo.toml @@ -0,0 +1,82 @@ +[package] +authors = ["Quentin de Quelen ", "Clément Renault "] +description = "MeiliSearch HTTP server" +edition = "2018" +license = "MIT" +name = "meilisearch-http" +version = "0.17.0" +[[bin]] +name = "meilisearch" +path = "src/main.rs" + +[build-dependencies] +vergen = "3.1.0" + +[dependencies] +actix-cors = { path = "../../actix-extras/actix-cors" } +actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } +actix-service = "2.0.0-beta.4" +actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } +#actix-web = { version = "3", features = ["rustls"] } +anyhow = "1.0.36" +async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } +byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } +bytes = "0.6.0" +chrono = { version = "0.4.19", features = ["serde"] } +crossbeam-channel = "0.5.0" +env_logger = "0.8.2" +flate2 = "1.0.19" +fst = "0.4.5" +futures = "0.3.7" +futures-util = "0.3.8" +grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } +heed = "0.10.6" +http = "0.2.1" +indexmap = { version = "1.3.2", features = ["serde-1"] } +log = "0.4.8" +main_error = "0.1.0" +meilisearch-error = { path = "../meilisearch-error" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } +memmap = "0.7.0" +milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } +mime = "0.3.16" +once_cell = "1.5.2" +rand = "0.7.3" +rayon = "1.5.0" +regex = "1.4.2" +rustls = "0.19" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0.59", features = ["preserve_order"] } +sha2 = "0.9.1" +siphasher = "0.3.2" +slice-group-by = "0.2.6" +structopt = "0.3.20" +tar = "0.4.29" +tempfile = "3.1.0" +tokio = { version = "1", features = ["full"] } +dashmap = "4.0.2" +uuid = "0.8.2" +itertools = "0.10.0" +either = "1.6.1" +async-trait = "0.1.42" +thiserror = "1.0.24" +async-stream = "0.3.0" + +[dependencies.sentry] +default-features = false +features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] +optional = true +version = "0.18.1" + + +[dev-dependencies] +serde_url_params = "0.2.0" +tempdir = "0.3.7" +assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } +actix-rt = "2.1.0" + +[features] +default = ["sentry"] + +[target.'cfg(unix)'.dependencies] +jemallocator = "0.3.2" diff --git a/build.rs b/meilisearch-http/build.rs similarity index 100% rename from build.rs rename to meilisearch-http/build.rs diff --git a/public/bulma.min.css b/meilisearch-http/public/bulma.min.css similarity index 100% rename from public/bulma.min.css rename to meilisearch-http/public/bulma.min.css diff --git a/public/interface.html b/meilisearch-http/public/interface.html similarity index 100% rename from public/interface.html rename to meilisearch-http/public/interface.html diff --git a/src/analytics.rs b/meilisearch-http/src/analytics.rs similarity index 100% rename from src/analytics.rs rename to meilisearch-http/src/analytics.rs diff --git a/src/data/mod.rs b/meilisearch-http/src/data/mod.rs similarity index 100% rename from src/data/mod.rs rename to meilisearch-http/src/data/mod.rs diff --git a/src/data/search.rs b/meilisearch-http/src/data/search.rs similarity index 100% rename from src/data/search.rs rename to meilisearch-http/src/data/search.rs diff --git a/src/data/updates.rs b/meilisearch-http/src/data/updates.rs similarity index 100% rename from src/data/updates.rs rename to meilisearch-http/src/data/updates.rs diff --git a/src/dump.rs b/meilisearch-http/src/dump.rs similarity index 100% rename from src/dump.rs rename to meilisearch-http/src/dump.rs diff --git a/src/error.rs b/meilisearch-http/src/error.rs similarity index 100% rename from src/error.rs rename to meilisearch-http/src/error.rs diff --git a/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs similarity index 100% rename from src/helpers/authentication.rs rename to meilisearch-http/src/helpers/authentication.rs diff --git a/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs similarity index 100% rename from src/helpers/compression.rs rename to meilisearch-http/src/helpers/compression.rs diff --git a/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs similarity index 100% rename from src/helpers/mod.rs rename to meilisearch-http/src/helpers/mod.rs diff --git a/src/helpers/normalize_path.rs b/meilisearch-http/src/helpers/normalize_path.rs similarity index 100% rename from src/helpers/normalize_path.rs rename to meilisearch-http/src/helpers/normalize_path.rs diff --git a/src/index/mod.rs b/meilisearch-http/src/index/mod.rs similarity index 100% rename from src/index/mod.rs rename to meilisearch-http/src/index/mod.rs diff --git a/src/index/search.rs b/meilisearch-http/src/index/search.rs similarity index 100% rename from src/index/search.rs rename to meilisearch-http/src/index/search.rs diff --git a/src/index/updates.rs b/meilisearch-http/src/index/updates.rs similarity index 100% rename from src/index/updates.rs rename to meilisearch-http/src/index/updates.rs diff --git a/src/index_controller/actor_index_controller/mod.rs b/meilisearch-http/src/index_controller/actor_index_controller/mod.rs similarity index 100% rename from src/index_controller/actor_index_controller/mod.rs rename to meilisearch-http/src/index_controller/actor_index_controller/mod.rs diff --git a/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs similarity index 100% rename from src/index_controller/index_actor.rs rename to meilisearch-http/src/index_controller/index_actor.rs diff --git a/src/index_controller/local_index_controller/index_store.rs b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs similarity index 100% rename from src/index_controller/local_index_controller/index_store.rs rename to meilisearch-http/src/index_controller/local_index_controller/index_store.rs diff --git a/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs similarity index 100% rename from src/index_controller/local_index_controller/mod.rs rename to meilisearch-http/src/index_controller/local_index_controller/mod.rs diff --git a/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs similarity index 100% rename from src/index_controller/mod.rs rename to meilisearch-http/src/index_controller/mod.rs diff --git a/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs similarity index 100% rename from src/index_controller/update_actor.rs rename to meilisearch-http/src/index_controller/update_actor.rs diff --git a/src/index_controller/update_handler.rs b/meilisearch-http/src/index_controller/update_handler.rs similarity index 100% rename from src/index_controller/update_handler.rs rename to meilisearch-http/src/index_controller/update_handler.rs diff --git a/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs similarity index 100% rename from src/index_controller/update_store.rs rename to meilisearch-http/src/index_controller/update_store.rs diff --git a/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs similarity index 100% rename from src/index_controller/updates.rs rename to meilisearch-http/src/index_controller/updates.rs diff --git a/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs similarity index 100% rename from src/index_controller/uuid_resolver.rs rename to meilisearch-http/src/index_controller/uuid_resolver.rs diff --git a/src/lib.rs b/meilisearch-http/src/lib.rs similarity index 100% rename from src/lib.rs rename to meilisearch-http/src/lib.rs diff --git a/src/main.rs b/meilisearch-http/src/main.rs similarity index 100% rename from src/main.rs rename to meilisearch-http/src/main.rs diff --git a/src/option.rs b/meilisearch-http/src/option.rs similarity index 100% rename from src/option.rs rename to meilisearch-http/src/option.rs diff --git a/src/routes/document.rs b/meilisearch-http/src/routes/document.rs similarity index 100% rename from src/routes/document.rs rename to meilisearch-http/src/routes/document.rs diff --git a/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs similarity index 100% rename from src/routes/dump.rs rename to meilisearch-http/src/routes/dump.rs diff --git a/src/routes/health.rs b/meilisearch-http/src/routes/health.rs similarity index 100% rename from src/routes/health.rs rename to meilisearch-http/src/routes/health.rs diff --git a/src/routes/index.rs b/meilisearch-http/src/routes/index.rs similarity index 100% rename from src/routes/index.rs rename to meilisearch-http/src/routes/index.rs diff --git a/src/routes/key.rs b/meilisearch-http/src/routes/key.rs similarity index 100% rename from src/routes/key.rs rename to meilisearch-http/src/routes/key.rs diff --git a/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs similarity index 100% rename from src/routes/mod.rs rename to meilisearch-http/src/routes/mod.rs diff --git a/src/routes/search.rs b/meilisearch-http/src/routes/search.rs similarity index 100% rename from src/routes/search.rs rename to meilisearch-http/src/routes/search.rs diff --git a/src/routes/settings/attributes_for_faceting.rs b/meilisearch-http/src/routes/settings/attributes_for_faceting.rs similarity index 100% rename from src/routes/settings/attributes_for_faceting.rs rename to meilisearch-http/src/routes/settings/attributes_for_faceting.rs diff --git a/src/routes/settings/displayed_attributes.rs b/meilisearch-http/src/routes/settings/displayed_attributes.rs similarity index 100% rename from src/routes/settings/displayed_attributes.rs rename to meilisearch-http/src/routes/settings/displayed_attributes.rs diff --git a/src/routes/settings/distinct_attributes.rs b/meilisearch-http/src/routes/settings/distinct_attributes.rs similarity index 100% rename from src/routes/settings/distinct_attributes.rs rename to meilisearch-http/src/routes/settings/distinct_attributes.rs diff --git a/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs similarity index 100% rename from src/routes/settings/mod.rs rename to meilisearch-http/src/routes/settings/mod.rs diff --git a/src/routes/settings/ranking_rules.rs b/meilisearch-http/src/routes/settings/ranking_rules.rs similarity index 100% rename from src/routes/settings/ranking_rules.rs rename to meilisearch-http/src/routes/settings/ranking_rules.rs diff --git a/src/routes/settings/searchable_attributes.rs b/meilisearch-http/src/routes/settings/searchable_attributes.rs similarity index 100% rename from src/routes/settings/searchable_attributes.rs rename to meilisearch-http/src/routes/settings/searchable_attributes.rs diff --git a/src/routes/settings/stop_words.rs b/meilisearch-http/src/routes/settings/stop_words.rs similarity index 100% rename from src/routes/settings/stop_words.rs rename to meilisearch-http/src/routes/settings/stop_words.rs diff --git a/src/routes/settings/synonyms.rs b/meilisearch-http/src/routes/settings/synonyms.rs similarity index 100% rename from src/routes/settings/synonyms.rs rename to meilisearch-http/src/routes/settings/synonyms.rs diff --git a/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs similarity index 100% rename from src/routes/stats.rs rename to meilisearch-http/src/routes/stats.rs diff --git a/src/routes/stop_words.rs b/meilisearch-http/src/routes/stop_words.rs similarity index 100% rename from src/routes/stop_words.rs rename to meilisearch-http/src/routes/stop_words.rs diff --git a/src/routes/synonym.rs b/meilisearch-http/src/routes/synonym.rs similarity index 100% rename from src/routes/synonym.rs rename to meilisearch-http/src/routes/synonym.rs diff --git a/src/snapshot.rs b/meilisearch-http/src/snapshot.rs similarity index 100% rename from src/snapshot.rs rename to meilisearch-http/src/snapshot.rs diff --git a/tests/assets/test_set.json b/meilisearch-http/tests/assets/test_set.json similarity index 100% rename from tests/assets/test_set.json rename to meilisearch-http/tests/assets/test_set.json diff --git a/tests/common/index.rs b/meilisearch-http/tests/common/index.rs similarity index 100% rename from tests/common/index.rs rename to meilisearch-http/tests/common/index.rs diff --git a/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs similarity index 100% rename from tests/common/mod.rs rename to meilisearch-http/tests/common/mod.rs diff --git a/tests/common/server.rs b/meilisearch-http/tests/common/server.rs similarity index 100% rename from tests/common/server.rs rename to meilisearch-http/tests/common/server.rs diff --git a/tests/common/service.rs b/meilisearch-http/tests/common/service.rs similarity index 100% rename from tests/common/service.rs rename to meilisearch-http/tests/common/service.rs diff --git a/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs similarity index 100% rename from tests/documents/add_documents.rs rename to meilisearch-http/tests/documents/add_documents.rs diff --git a/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs similarity index 100% rename from tests/documents/delete_documents.rs rename to meilisearch-http/tests/documents/delete_documents.rs diff --git a/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs similarity index 100% rename from tests/documents/get_documents.rs rename to meilisearch-http/tests/documents/get_documents.rs diff --git a/tests/documents/mod.rs b/meilisearch-http/tests/documents/mod.rs similarity index 100% rename from tests/documents/mod.rs rename to meilisearch-http/tests/documents/mod.rs diff --git a/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs similarity index 100% rename from tests/index/create_index.rs rename to meilisearch-http/tests/index/create_index.rs diff --git a/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs similarity index 100% rename from tests/index/delete_index.rs rename to meilisearch-http/tests/index/delete_index.rs diff --git a/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs similarity index 100% rename from tests/index/get_index.rs rename to meilisearch-http/tests/index/get_index.rs diff --git a/tests/index/mod.rs b/meilisearch-http/tests/index/mod.rs similarity index 100% rename from tests/index/mod.rs rename to meilisearch-http/tests/index/mod.rs diff --git a/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs similarity index 100% rename from tests/index/update_index.rs rename to meilisearch-http/tests/index/update_index.rs diff --git a/tests/integration.rs b/meilisearch-http/tests/integration.rs similarity index 100% rename from tests/integration.rs rename to meilisearch-http/tests/integration.rs diff --git a/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs similarity index 100% rename from tests/search/mod.rs rename to meilisearch-http/tests/search/mod.rs diff --git a/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs similarity index 100% rename from tests/settings/get_settings.rs rename to meilisearch-http/tests/settings/get_settings.rs diff --git a/tests/settings/mod.rs b/meilisearch-http/tests/settings/mod.rs similarity index 100% rename from tests/settings/mod.rs rename to meilisearch-http/tests/settings/mod.rs diff --git a/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs similarity index 100% rename from tests/updates/mod.rs rename to meilisearch-http/tests/updates/mod.rs From 0cd8869349b5cd96242b112b887e3d7a863af8d9 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 14:43:10 +0100 Subject: [PATCH 151/522] update relevant changes from master --- .dockerignore | 5 ++ .github/workflows/create_artifacts.yml | 38 +++++++++ .github/workflows/publish_to_docker.yml | 20 +++++ .github/workflows/rust.yml | 6 +- LICENSE | 21 +++++ .../tests/assets/dumps/v1/metadata.json | 12 +++ .../assets/dumps/v1/test/documents.jsonl | 77 +++++++++++++++++++ .../tests/assets/dumps/v1/test/settings.json | 59 ++++++++++++++ .../tests/assets/dumps/v1/test/updates.jsonl | 2 + meilisearch-http/tests/common/index.rs | 4 +- meilisearch-http/tests/common/server.rs | 7 +- meilisearch-http/tests/common/service.rs | 14 ++-- .../tests/documents/add_documents.rs | 16 ++++ meilisearch-http/tests/index/create_index.rs | 4 +- .../tests/settings/get_settings.rs | 15 +++- meilisearch-http/tests/updates/mod.rs | 2 - 16 files changed, 276 insertions(+), 26 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/create_artifacts.yml create mode 100644 .github/workflows/publish_to_docker.yml create mode 100644 LICENSE create mode 100644 meilisearch-http/tests/assets/dumps/v1/metadata.json create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/settings.json create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..364510117 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +target +Dockerfile +.dockerignore +.git +.gitignore diff --git a/.github/workflows/create_artifacts.yml b/.github/workflows/create_artifacts.yml new file mode 100644 index 000000000..94378ba83 --- /dev/null +++ b/.github/workflows/create_artifacts.yml @@ -0,0 +1,38 @@ +name: Create artifacts + +on: + push: + tags: + - v*-alpha.* + +jobs: + nightly: + name: Build Nighlty ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + artifact_name: meilisearch + asset_name: meilisearch-alpha-linux-amd64 + - os: macos-latest + artifact_name: meilisearch + asset_name: meilisearch-alpha-macos-amd64 + - os: windows-latest + artifact_name: meilisearch.exe + asset_name: meilisearch-alpha-windows-amd64.exe + steps: + - uses: hecrj/setup-rust-action@master + with: + rust-version: stable + - uses: actions/checkout@v1 + - name: Build + run: cargo build --release --locked + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v1-release + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: target/release/${{ matrix.artifact_name }} + asset_name: ${{ matrix.asset_name }} + tag: ${{ github.ref }} diff --git a/.github/workflows/publish_to_docker.yml b/.github/workflows/publish_to_docker.yml new file mode 100644 index 000000000..d724f7253 --- /dev/null +++ b/.github/workflows/publish_to_docker.yml @@ -0,0 +1,20 @@ +name: Publish to dockerhub + +on: + push: + tags: + - v*-alpha.* + +jobs: + publish: + name: Publishing to dockerhub + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Publish to Registry + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: getmeili/meilisearch + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + tag_names: true diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3c13d1be2..ea4b41a6a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,10 +1,6 @@ name: Rust -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: push env: CARGO_TERM_COLOR: always diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..87471adb1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Meili + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/meilisearch-http/tests/assets/dumps/v1/metadata.json b/meilisearch-http/tests/assets/dumps/v1/metadata.json new file mode 100644 index 000000000..6fe302324 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/metadata.json @@ -0,0 +1,12 @@ +{ + "indices": [{ + "uid": "test", + "primaryKey": "id" + }, { + "uid": "test2", + "primaryKey": "test2_id" + } + ], + "dbVersion": "0.13.0", + "dumpVersion": "1" +} diff --git a/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl new file mode 100644 index 000000000..7af80f342 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl @@ -0,0 +1,77 @@ +{"id":0,"isActive":false,"balance":"$2,668.55","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Lucas Hess","gender":"male","email":"lucashess@chorizon.com","phone":"+1 (998) 478-2597","address":"412 Losee Terrace, Blairstown, Georgia, 2825","about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n","registered":"2016-06-21T09:30:25 -02:00","latitude":-44.174957,"longitude":-145.725388,"tags":["bug","bug"]} +{"id":1,"isActive":true,"balance":"$1,706.13","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Cherry Orr","gender":"female","email":"cherryorr@chorizon.com","phone":"+1 (995) 479-3174","address":"442 Beverly Road, Ventress, New Mexico, 3361","about":"Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n","registered":"2020-03-18T11:12:21 -01:00","latitude":-24.356932,"longitude":27.184808,"tags":["new issue","bug"]} +{"id":2,"isActive":true,"balance":"$2,467.47","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Patricia Goff","gender":"female","email":"patriciagoff@chorizon.com","phone":"+1 (864) 463-2277","address":"866 Hornell Loop, Cresaptown, Ohio, 1700","about":"Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n","registered":"2014-10-28T12:59:30 -01:00","latitude":-64.008555,"longitude":11.867098,"tags":["good first issue"]} +{"id":3,"isActive":true,"balance":"$3,344.40","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Adeline Flynn","gender":"female","email":"adelineflynn@chorizon.com","phone":"+1 (994) 600-2840","address":"428 Paerdegat Avenue, Hollymead, Pennsylvania, 948","about":"Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n","registered":"2014-03-27T06:24:45 -01:00","latitude":-74.485173,"longitude":-11.059859,"tags":["bug","good first issue","wontfix","new issue"]} +{"id":4,"isActive":false,"balance":"$2,575.78","picture":"http://placehold.it/32x32","age":39,"color":"Green","name":"Mariana Pacheco","gender":"female","email":"marianapacheco@chorizon.com","phone":"+1 (820) 414-2223","address":"664 Rapelye Street, Faywood, California, 7320","about":"Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n","registered":"2015-09-02T03:23:35 -02:00","latitude":75.763501,"longitude":-78.777124,"tags":["new issue"]} +{"id":5,"isActive":true,"balance":"$3,793.09","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Warren Watson","gender":"male","email":"warrenwatson@chorizon.com","phone":"+1 (807) 583-2427","address":"671 Prince Street, Faxon, Connecticut, 4275","about":"Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n","registered":"2017-06-04T06:02:17 -02:00","latitude":29.979223,"longitude":25.358943,"tags":["wontfix","wontfix","wontfix"]} +{"id":6,"isActive":true,"balance":"$2,919.70","picture":"http://placehold.it/32x32","age":20,"color":"blue","name":"Shelia Berry","gender":"female","email":"sheliaberry@chorizon.com","phone":"+1 (853) 511-2651","address":"437 Forrest Street, Coventry, Illinois, 2056","about":"Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n","registered":"2018-07-11T02:45:01 -02:00","latitude":54.815991,"longitude":-118.690609,"tags":["good first issue","bug","wontfix","new issue"]} +{"id":7,"isActive":true,"balance":"$1,349.50","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Chrystal Boyd","gender":"female","email":"chrystalboyd@chorizon.com","phone":"+1 (936) 563-2802","address":"670 Croton Loop, Sussex, Florida, 4692","about":"Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n","registered":"2016-11-01T07:36:04 -01:00","latitude":-24.711933,"longitude":147.246705,"tags":[]} +{"id":8,"isActive":false,"balance":"$3,999.56","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Martin Porter","gender":"male","email":"martinporter@chorizon.com","phone":"+1 (895) 580-2304","address":"577 Regent Place, Aguila, Guam, 6554","about":"Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n","registered":"2014-09-20T02:08:30 -02:00","latitude":-88.344273,"longitude":37.964466,"tags":[]} +{"id":9,"isActive":true,"balance":"$3,729.71","picture":"http://placehold.it/32x32","age":26,"color":"blue","name":"Kelli Mendez","gender":"female","email":"kellimendez@chorizon.com","phone":"+1 (936) 401-2236","address":"242 Caton Place, Grazierville, Alabama, 3968","about":"Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n","registered":"2018-05-04T10:35:30 -02:00","latitude":49.37551,"longitude":41.872323,"tags":["new issue","new issue"]} +{"id":10,"isActive":false,"balance":"$1,127.47","picture":"http://placehold.it/32x32","age":27,"color":"blue","name":"Maddox Johns","gender":"male","email":"maddoxjohns@chorizon.com","phone":"+1 (892) 470-2357","address":"756 Beard Street, Avalon, Louisiana, 114","about":"Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n","registered":"2016-04-22T06:41:25 -02:00","latitude":66.640229,"longitude":-17.222666,"tags":["new issue","good first issue","good first issue","new issue"]} +{"id":11,"isActive":true,"balance":"$1,351.43","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Evans Wagner","gender":"male","email":"evanswagner@chorizon.com","phone":"+1 (889) 496-2332","address":"118 Monaco Place, Lutsen, Delaware, 6209","about":"Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n","registered":"2016-10-27T01:26:31 -02:00","latitude":-77.673222,"longitude":-142.657214,"tags":["good first issue","good first issue"]} +{"id":12,"isActive":false,"balance":"$3,394.96","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Aida Kirby","gender":"female","email":"aidakirby@chorizon.com","phone":"+1 (942) 532-2325","address":"797 Engert Avenue, Wilsonia, Idaho, 6532","about":"Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n","registered":"2018-06-18T04:39:57 -02:00","latitude":-58.062041,"longitude":34.999254,"tags":["new issue","wontfix","bug","new issue"]} +{"id":13,"isActive":true,"balance":"$2,812.62","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Nelda Burris","gender":"female","email":"neldaburris@chorizon.com","phone":"+1 (813) 600-2576","address":"160 Opal Court, Fowlerville, Tennessee, 2170","about":"Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n","registered":"2015-08-15T12:39:53 -02:00","latitude":66.6871,"longitude":179.549488,"tags":["wontfix"]} +{"id":14,"isActive":true,"balance":"$1,718.33","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Jennifer Hart","gender":"female","email":"jenniferhart@chorizon.com","phone":"+1 (850) 537-2513","address":"124 Veranda Place, Nash, Utah, 985","about":"Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n","registered":"2016-09-04T11:46:59 -02:00","latitude":-66.827751,"longitude":99.220079,"tags":["wontfix","bug","new issue","new issue"]} +{"id":15,"isActive":false,"balance":"$2,698.16","picture":"http://placehold.it/32x32","age":28,"color":"blue","name":"Aurelia Contreras","gender":"female","email":"aureliacontreras@chorizon.com","phone":"+1 (932) 442-3103","address":"655 Dwight Street, Grapeview, Palau, 8356","about":"Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n","registered":"2014-09-11T10:43:15 -02:00","latitude":-71.328973,"longitude":133.404895,"tags":["wontfix","bug","good first issue"]} +{"id":16,"isActive":true,"balance":"$3,303.25","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Estella Bass","gender":"female","email":"estellabass@chorizon.com","phone":"+1 (825) 436-2909","address":"435 Rockwell Place, Garberville, Wisconsin, 2230","about":"Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n","registered":"2017-11-23T09:32:09 -01:00","latitude":81.17014,"longitude":-145.262693,"tags":["new issue"]} +{"id":17,"isActive":false,"balance":"$3,579.20","picture":"http://placehold.it/32x32","age":25,"color":"brown","name":"Ortega Brennan","gender":"male","email":"ortegabrennan@chorizon.com","phone":"+1 (906) 526-2287","address":"440 Berry Street, Rivera, Maine, 1849","about":"Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n","registered":"2016-03-31T02:17:13 -02:00","latitude":-68.407524,"longitude":-113.642067,"tags":["new issue","wontfix"]} +{"id":18,"isActive":false,"balance":"$1,484.92","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Leonard Tillman","gender":"male","email":"leonardtillman@chorizon.com","phone":"+1 (864) 541-3456","address":"985 Provost Street, Charco, New Hampshire, 8632","about":"Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n","registered":"2018-05-06T08:21:27 -02:00","latitude":-8.581801,"longitude":-61.910062,"tags":["wontfix","new issue","bug","bug"]} +{"id":19,"isActive":true,"balance":"$3,572.55","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Dale Payne","gender":"male","email":"dalepayne@chorizon.com","phone":"+1 (814) 469-3499","address":"536 Dare Court, Ironton, Arkansas, 8605","about":"Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n","registered":"2019-10-11T01:01:33 -02:00","latitude":-18.280968,"longitude":-126.091797,"tags":["bug","wontfix","wontfix","wontfix"]} +{"id":20,"isActive":true,"balance":"$1,986.48","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Florence Long","gender":"female","email":"florencelong@chorizon.com","phone":"+1 (972) 557-3858","address":"519 Hendrickson Street, Templeton, Hawaii, 2389","about":"Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n","registered":"2016-05-02T09:18:59 -02:00","latitude":-27.110866,"longitude":-45.09445,"tags":[]} +{"id":21,"isActive":true,"balance":"$1,440.09","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Levy Whitley","gender":"male","email":"levywhitley@chorizon.com","phone":"+1 (911) 458-2411","address":"187 Thomas Street, Hachita, North Carolina, 2989","about":"Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n","registered":"2014-04-30T07:31:38 -02:00","latitude":-6.537315,"longitude":171.813536,"tags":["bug"]} +{"id":22,"isActive":false,"balance":"$2,938.57","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Bernard Mcfarland","gender":"male","email":"bernardmcfarland@chorizon.com","phone":"+1 (979) 442-3386","address":"409 Hall Street, Keyport, Federated States Of Micronesia, 7011","about":"Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n","registered":"2017-08-10T10:07:59 -02:00","latitude":63.766795,"longitude":68.177069,"tags":[]} +{"id":23,"isActive":true,"balance":"$1,678.49","picture":"http://placehold.it/32x32","age":31,"color":"brown","name":"Blanca Mcclain","gender":"female","email":"blancamcclain@chorizon.com","phone":"+1 (976) 439-2772","address":"176 Crooke Avenue, Valle, Virginia, 5373","about":"Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n","registered":"2015-10-12T11:57:28 -02:00","latitude":-8.944564,"longitude":-150.711709,"tags":["bug","wontfix","good first issue"]} +{"id":24,"isActive":true,"balance":"$2,276.87","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Espinoza Ford","gender":"male","email":"espinozaford@chorizon.com","phone":"+1 (945) 429-3975","address":"137 Bowery Street, Itmann, District Of Columbia, 1864","about":"Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n","registered":"2014-03-26T02:16:08 -01:00","latitude":-37.137666,"longitude":-51.811757,"tags":["wontfix","bug"]} +{"id":25,"isActive":true,"balance":"$3,973.43","picture":"http://placehold.it/32x32","age":29,"color":"Green","name":"Sykes Conley","gender":"male","email":"sykesconley@chorizon.com","phone":"+1 (851) 401-3916","address":"345 Grand Street, Woodlands, Missouri, 4461","about":"Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n","registered":"2015-09-12T06:03:56 -02:00","latitude":67.282955,"longitude":-64.341323,"tags":["wontfix"]} +{"id":26,"isActive":false,"balance":"$1,431.50","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Barlow Duran","gender":"male","email":"barlowduran@chorizon.com","phone":"+1 (995) 436-2562","address":"481 Everett Avenue, Allison, Nebraska, 3065","about":"Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n","registered":"2017-06-29T04:28:43 -02:00","latitude":-38.70606,"longitude":55.02816,"tags":["new issue"]} +{"id":27,"isActive":true,"balance":"$3,478.27","picture":"http://placehold.it/32x32","age":31,"color":"blue","name":"Schwartz Morgan","gender":"male","email":"schwartzmorgan@chorizon.com","phone":"+1 (861) 507-2067","address":"451 Lincoln Road, Fairlee, Washington, 2717","about":"Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n","registered":"2016-05-10T08:34:54 -02:00","latitude":-75.886403,"longitude":93.044471,"tags":["bug","bug","wontfix","wontfix"]} +{"id":28,"isActive":true,"balance":"$2,825.59","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Kristy Leon","gender":"female","email":"kristyleon@chorizon.com","phone":"+1 (948) 465-2563","address":"594 Macon Street, Floris, South Dakota, 3565","about":"Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n","registered":"2014-12-14T04:10:29 -01:00","latitude":-50.01615,"longitude":-68.908804,"tags":["wontfix","good first issue"]} +{"id":29,"isActive":false,"balance":"$3,028.03","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Ashley Pittman","gender":"male","email":"ashleypittman@chorizon.com","phone":"+1 (928) 507-3523","address":"646 Adelphi Street, Clara, Colorado, 6056","about":"Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n","registered":"2016-01-07T10:40:48 -01:00","latitude":-58.766037,"longitude":-124.828485,"tags":["wontfix"]} +{"id":30,"isActive":true,"balance":"$2,021.11","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Stacy Espinoza","gender":"female","email":"stacyespinoza@chorizon.com","phone":"+1 (999) 487-3253","address":"931 Alabama Avenue, Bangor, Alaska, 8215","about":"Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n","registered":"2014-07-16T06:15:53 -02:00","latitude":41.560197,"longitude":177.697,"tags":["new issue","new issue","bug"]} +{"id":31,"isActive":false,"balance":"$3,609.82","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Vilma Garza","gender":"female","email":"vilmagarza@chorizon.com","phone":"+1 (944) 585-2021","address":"565 Tech Place, Sedley, Puerto Rico, 858","about":"Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n","registered":"2017-06-30T07:43:52 -02:00","latitude":-12.574889,"longitude":-54.771186,"tags":["new issue","wontfix","wontfix"]} +{"id":32,"isActive":false,"balance":"$2,882.34","picture":"http://placehold.it/32x32","age":38,"color":"brown","name":"June Dunlap","gender":"female","email":"junedunlap@chorizon.com","phone":"+1 (997) 504-2937","address":"353 Cozine Avenue, Goodville, Indiana, 1438","about":"Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n","registered":"2016-08-23T08:54:11 -02:00","latitude":-27.883363,"longitude":-163.919683,"tags":["new issue","new issue","bug","wontfix"]} +{"id":33,"isActive":true,"balance":"$3,556.54","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Cecilia Greer","gender":"female","email":"ceciliagreer@chorizon.com","phone":"+1 (977) 573-3498","address":"696 Withers Street, Lydia, Oklahoma, 3220","about":"Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n","registered":"2017-01-13T11:30:12 -01:00","latitude":60.467215,"longitude":84.684575,"tags":["wontfix","good first issue","good first issue","wontfix"]} +{"id":34,"isActive":true,"balance":"$1,413.35","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Mckay Schroeder","gender":"male","email":"mckayschroeder@chorizon.com","phone":"+1 (816) 480-3657","address":"958 Miami Court, Rehrersburg, Northern Mariana Islands, 567","about":"Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n","registered":"2016-02-08T04:50:15 -01:00","latitude":-72.413287,"longitude":-159.254371,"tags":["good first issue"]} +{"id":35,"isActive":true,"balance":"$2,306.53","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Sawyer Mccormick","gender":"male","email":"sawyermccormick@chorizon.com","phone":"+1 (829) 569-3012","address":"749 Apollo Street, Eastvale, Texas, 7373","about":"Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n","registered":"2019-11-30T11:53:23 -01:00","latitude":-48.978194,"longitude":110.950191,"tags":["good first issue","new issue","new issue","bug"]} +{"id":36,"isActive":false,"balance":"$1,844.54","picture":"http://placehold.it/32x32","age":37,"color":"brown","name":"Barbra Valenzuela","gender":"female","email":"barbravalenzuela@chorizon.com","phone":"+1 (992) 512-2649","address":"617 Schenck Court, Reinerton, Michigan, 2908","about":"Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n","registered":"2019-03-29T01:59:31 -01:00","latitude":45.193723,"longitude":-12.486778,"tags":["new issue","new issue","wontfix","wontfix"]} +{"id":37,"isActive":false,"balance":"$3,469.82","picture":"http://placehold.it/32x32","age":39,"color":"brown","name":"Opal Weiss","gender":"female","email":"opalweiss@chorizon.com","phone":"+1 (809) 400-3079","address":"535 Bogart Street, Frizzleburg, Arizona, 5222","about":"Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n","registered":"2019-09-04T07:22:28 -02:00","latitude":72.50376,"longitude":61.656435,"tags":["bug","bug","good first issue","good first issue"]} +{"id":38,"isActive":true,"balance":"$1,992.38","picture":"http://placehold.it/32x32","age":40,"color":"Green","name":"Christina Short","gender":"female","email":"christinashort@chorizon.com","phone":"+1 (884) 589-2705","address":"594 Willmohr Street, Dexter, Montana, 660","about":"Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n","registered":"2014-01-21T09:31:56 -01:00","latitude":-42.762739,"longitude":77.052349,"tags":["bug","new issue"]} +{"id":39,"isActive":false,"balance":"$1,722.85","picture":"http://placehold.it/32x32","age":29,"color":"brown","name":"Golden Horton","gender":"male","email":"goldenhorton@chorizon.com","phone":"+1 (903) 426-2489","address":"191 Schenck Avenue, Mayfair, North Dakota, 5000","about":"Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n","registered":"2015-08-19T02:56:41 -02:00","latitude":69.922534,"longitude":9.881433,"tags":["bug"]} +{"id":40,"isActive":false,"balance":"$1,656.54","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Stafford Emerson","gender":"male","email":"staffordemerson@chorizon.com","phone":"+1 (992) 455-2573","address":"523 Thornton Street, Conway, Vermont, 6331","about":"Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n","registered":"2019-02-16T04:07:08 -01:00","latitude":-29.143111,"longitude":-57.207703,"tags":["wontfix","good first issue","good first issue"]} +{"id":41,"isActive":false,"balance":"$1,861.56","picture":"http://placehold.it/32x32","age":21,"color":"brown","name":"Salinas Gamble","gender":"male","email":"salinasgamble@chorizon.com","phone":"+1 (901) 525-2373","address":"991 Nostrand Avenue, Kansas, Mississippi, 6756","about":"Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n","registered":"2017-08-21T05:47:53 -02:00","latitude":-22.593819,"longitude":-63.613004,"tags":["good first issue","bug","bug","wontfix"]} +{"id":42,"isActive":true,"balance":"$3,179.74","picture":"http://placehold.it/32x32","age":34,"color":"brown","name":"Graciela Russell","gender":"female","email":"gracielarussell@chorizon.com","phone":"+1 (893) 464-3951","address":"361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713","about":"Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n","registered":"2015-05-18T09:52:56 -02:00","latitude":-14.634444,"longitude":12.931783,"tags":["wontfix","bug","wontfix"]} +{"id":43,"isActive":true,"balance":"$1,777.38","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Arnold Bender","gender":"male","email":"arnoldbender@chorizon.com","phone":"+1 (945) 581-3808","address":"781 Lorraine Street, Gallina, American Samoa, 1832","about":"Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n","registered":"2018-12-23T02:26:30 -01:00","latitude":41.208579,"longitude":51.948925,"tags":["bug","good first issue","good first issue","wontfix"]} +{"id":44,"isActive":true,"balance":"$2,893.45","picture":"http://placehold.it/32x32","age":22,"color":"Green","name":"Joni Spears","gender":"female","email":"jonispears@chorizon.com","phone":"+1 (916) 565-2124","address":"307 Harwood Place, Canterwood, Maryland, 2047","about":"Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n","registered":"2015-03-01T12:38:28 -01:00","latitude":8.19071,"longitude":146.323808,"tags":["wontfix","new issue","good first issue","good first issue"]} +{"id":45,"isActive":true,"balance":"$2,830.36","picture":"http://placehold.it/32x32","age":20,"color":"brown","name":"Irene Bennett","gender":"female","email":"irenebennett@chorizon.com","phone":"+1 (904) 431-2211","address":"353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686","about":"Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n","registered":"2018-04-17T05:18:51 -02:00","latitude":-36.435177,"longitude":-127.552573,"tags":["bug","wontfix"]} +{"id":46,"isActive":true,"balance":"$1,348.04","picture":"http://placehold.it/32x32","age":34,"color":"Green","name":"Lawson Curtis","gender":"male","email":"lawsoncurtis@chorizon.com","phone":"+1 (896) 532-2172","address":"942 Gerritsen Avenue, Southmont, Kansas, 8915","about":"Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n","registered":"2016-08-23T01:41:09 -02:00","latitude":-48.783539,"longitude":20.492944,"tags":[]} +{"id":47,"isActive":true,"balance":"$1,132.41","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goff May","gender":"male","email":"goffmay@chorizon.com","phone":"+1 (859) 453-3415","address":"225 Rutledge Street, Boonville, Massachusetts, 4081","about":"Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n","registered":"2014-10-25T07:32:30 -02:00","latitude":13.079225,"longitude":76.215086,"tags":["bug"]} +{"id":48,"isActive":true,"balance":"$1,201.87","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goodman Becker","gender":"male","email":"goodmanbecker@chorizon.com","phone":"+1 (825) 470-3437","address":"388 Seigel Street, Sisquoc, Kentucky, 8231","about":"Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n","registered":"2019-09-05T04:49:03 -02:00","latitude":-23.792094,"longitude":-13.621221,"tags":["bug","bug","wontfix","new issue"]} +{"id":49,"isActive":true,"balance":"$1,476.39","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Maureen Dale","gender":"female","email":"maureendale@chorizon.com","phone":"+1 (984) 538-3684","address":"817 Newton Street, Bannock, Wyoming, 1468","about":"Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n","registered":"2018-04-26T06:04:40 -02:00","latitude":-64.196802,"longitude":-117.396238,"tags":["wontfix"]} +{"id":50,"isActive":true,"balance":"$1,947.08","picture":"http://placehold.it/32x32","age":21,"color":"Green","name":"Guerra Mcintyre","gender":"male","email":"guerramcintyre@chorizon.com","phone":"+1 (951) 536-2043","address":"423 Lombardy Street, Stewart, West Virginia, 908","about":"Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n","registered":"2015-07-16T05:11:42 -02:00","latitude":79.733743,"longitude":-20.602356,"tags":["bug","good first issue","good first issue"]} +{"id":51,"isActive":true,"balance":"$2,960.90","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Key Cervantes","gender":"male","email":"keycervantes@chorizon.com","phone":"+1 (931) 474-3865","address":"410 Barbey Street, Vernon, Oregon, 2328","about":"Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n","registered":"2019-12-15T12:13:35 -01:00","latitude":47.627647,"longitude":117.049918,"tags":["new issue"]} +{"id":52,"isActive":false,"balance":"$1,884.02","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Karen Nelson","gender":"female","email":"karennelson@chorizon.com","phone":"+1 (993) 528-3607","address":"930 Frank Court, Dunbar, New York, 8810","about":"Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n","registered":"2014-06-23T09:21:44 -02:00","latitude":-59.059033,"longitude":76.565373,"tags":["new issue","bug"]} +{"id":53,"isActive":true,"balance":"$3,559.55","picture":"http://placehold.it/32x32","age":32,"color":"brown","name":"Caitlin Burnett","gender":"female","email":"caitlinburnett@chorizon.com","phone":"+1 (945) 480-2796","address":"516 Senator Street, Emory, Iowa, 4145","about":"In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n","registered":"2019-01-09T02:26:31 -01:00","latitude":-82.774237,"longitude":42.316194,"tags":["bug","good first issue"]} +{"id":54,"isActive":true,"balance":"$2,113.29","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Richards Walls","gender":"male","email":"richardswalls@chorizon.com","phone":"+1 (865) 517-2982","address":"959 Brightwater Avenue, Stevens, Nevada, 2968","about":"Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n","registered":"2014-09-25T06:51:22 -02:00","latitude":80.09202,"longitude":87.49759,"tags":["wontfix","wontfix","bug"]} +{"id":55,"isActive":true,"balance":"$1,977.66","picture":"http://placehold.it/32x32","age":36,"color":"brown","name":"Combs Stanley","gender":"male","email":"combsstanley@chorizon.com","phone":"+1 (827) 419-2053","address":"153 Beverley Road, Siglerville, South Carolina, 3666","about":"Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n","registered":"2019-08-22T07:53:15 -02:00","latitude":78.386181,"longitude":143.661058,"tags":[]} +{"id":56,"isActive":false,"balance":"$3,886.12","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Tucker Barry","gender":"male","email":"tuckerbarry@chorizon.com","phone":"+1 (808) 544-3433","address":"805 Jamaica Avenue, Cornfields, Minnesota, 3689","about":"Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n","registered":"2016-08-29T07:28:00 -02:00","latitude":71.701551,"longitude":9.903068,"tags":[]} +{"id":57,"isActive":false,"balance":"$1,844.56","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Kaitlin Conner","gender":"female","email":"kaitlinconner@chorizon.com","phone":"+1 (862) 467-2666","address":"501 Knight Court, Joppa, Rhode Island, 274","about":"Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n","registered":"2019-05-30T06:38:24 -02:00","latitude":15.613464,"longitude":171.965629,"tags":[]} +{"id":58,"isActive":true,"balance":"$2,876.10","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Mamie Fischer","gender":"female","email":"mamiefischer@chorizon.com","phone":"+1 (948) 545-3901","address":"599 Hunterfly Place, Haena, Georgia, 6005","about":"Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n","registered":"2019-05-27T05:07:10 -02:00","latitude":70.915079,"longitude":-48.813584,"tags":["bug","wontfix","wontfix","good first issue"]} +{"id":59,"isActive":true,"balance":"$1,921.58","picture":"http://placehold.it/32x32","age":31,"color":"Green","name":"Harper Carson","gender":"male","email":"harpercarson@chorizon.com","phone":"+1 (912) 430-3243","address":"883 Dennett Place, Knowlton, New Mexico, 9219","about":"Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n","registered":"2019-12-07T07:33:15 -01:00","latitude":-60.812605,"longitude":-27.129016,"tags":["bug","new issue"]} +{"id":60,"isActive":true,"balance":"$1,770.93","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Jody Herrera","gender":"female","email":"jodyherrera@chorizon.com","phone":"+1 (890) 583-3222","address":"261 Jay Street, Strykersville, Ohio, 9248","about":"Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n","registered":"2016-05-21T01:00:02 -02:00","latitude":-36.846586,"longitude":131.156223,"tags":[]} +{"id":61,"isActive":false,"balance":"$2,813.41","picture":"http://placehold.it/32x32","age":37,"color":"Green","name":"Charles Castillo","gender":"male","email":"charlescastillo@chorizon.com","phone":"+1 (934) 467-2108","address":"675 Morton Street, Rew, Pennsylvania, 137","about":"Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n","registered":"2019-06-10T02:54:22 -02:00","latitude":-16.423202,"longitude":-146.293752,"tags":["new issue","new issue"]} +{"id":62,"isActive":true,"balance":"$3,341.35","picture":"http://placehold.it/32x32","age":33,"color":"blue","name":"Estelle Ramirez","gender":"female","email":"estelleramirez@chorizon.com","phone":"+1 (816) 459-2073","address":"636 Nolans Lane, Camptown, California, 7794","about":"Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n","registered":"2015-02-14T01:05:50 -01:00","latitude":-46.591249,"longitude":-83.385587,"tags":["good first issue","bug"]} +{"id":63,"isActive":true,"balance":"$2,478.30","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Knowles Hebert","gender":"male","email":"knowleshebert@chorizon.com","phone":"+1 (819) 409-2308","address":"361 Kathleen Court, Gratton, Connecticut, 7254","about":"Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n","registered":"2016-03-08T08:34:52 -01:00","latitude":71.042482,"longitude":152.460406,"tags":["good first issue","wontfix"]} +{"id":64,"isActive":false,"balance":"$2,559.09","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Thelma Mckenzie","gender":"female","email":"thelmamckenzie@chorizon.com","phone":"+1 (941) 596-2777","address":"202 Leonard Street, Riverton, Illinois, 8577","about":"Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n","registered":"2020-04-14T12:43:06 -02:00","latitude":16.026129,"longitude":105.464476,"tags":[]} +{"id":65,"isActive":true,"balance":"$1,025.08","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Carole Rowland","gender":"female","email":"carolerowland@chorizon.com","phone":"+1 (862) 558-3448","address":"941 Melba Court, Bluetown, Florida, 9555","about":"Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n","registered":"2014-12-01T05:55:35 -01:00","latitude":-0.191998,"longitude":43.389652,"tags":["wontfix"]} +{"id":66,"isActive":true,"balance":"$1,061.49","picture":"http://placehold.it/32x32","age":35,"color":"brown","name":"Higgins Aguilar","gender":"male","email":"higginsaguilar@chorizon.com","phone":"+1 (911) 540-3791","address":"132 Sackman Street, Layhill, Guam, 8729","about":"Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n","registered":"2015-04-05T02:10:07 -02:00","latitude":74.702813,"longitude":151.314972,"tags":["bug"]} +{"id":67,"isActive":true,"balance":"$3,510.14","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Ilene Gillespie","gender":"female","email":"ilenegillespie@chorizon.com","phone":"+1 (937) 575-2676","address":"835 Lake Street, Naomi, Alabama, 4131","about":"Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n","registered":"2015-06-28T09:41:45 -02:00","latitude":71.573342,"longitude":-95.295989,"tags":["wontfix","wontfix"]} +{"id":68,"isActive":false,"balance":"$1,539.98","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Angelina Dyer","gender":"female","email":"angelinadyer@chorizon.com","phone":"+1 (948) 574-3949","address":"575 Division Place, Gorham, Louisiana, 3458","about":"Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n","registered":"2014-07-08T06:34:36 -02:00","latitude":-85.649593,"longitude":66.126018,"tags":["good first issue"]} +{"id":69,"isActive":true,"balance":"$3,367.69","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Marks Burt","gender":"male","email":"marksburt@chorizon.com","phone":"+1 (895) 497-3138","address":"819 Village Road, Wadsworth, Delaware, 6099","about":"Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n","registered":"2014-08-31T06:12:18 -02:00","latitude":26.854112,"longitude":-143.313948,"tags":["good first issue"]} +{"id":70,"isActive":false,"balance":"$3,755.72","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Glass Perkins","gender":"male","email":"glassperkins@chorizon.com","phone":"+1 (923) 486-3725","address":"899 Roosevelt Court, Belleview, Idaho, 1737","about":"Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n","registered":"2015-05-22T05:44:33 -02:00","latitude":54.27147,"longitude":-65.065604,"tags":["wontfix"]} +{"id":71,"isActive":true,"balance":"$3,381.63","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Candace Sawyer","gender":"female","email":"candacesawyer@chorizon.com","phone":"+1 (830) 404-2636","address":"334 Arkansas Drive, Bordelonville, Tennessee, 8449","about":"Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n","registered":"2014-04-04T08:45:00 -02:00","latitude":6.484262,"longitude":-37.054928,"tags":["new issue","new issue"]} +{"id":72,"isActive":true,"balance":"$1,640.98","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Hendricks Martinez","gender":"male","email":"hendricksmartinez@chorizon.com","phone":"+1 (857) 566-3245","address":"636 Agate Court, Newry, Utah, 3304","about":"Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n","registered":"2018-06-15T10:36:11 -02:00","latitude":86.746034,"longitude":10.347893,"tags":["new issue"]} +{"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]} +{"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]} +{"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]} +{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]} \ No newline at end of file diff --git a/meilisearch-http/tests/assets/dumps/v1/test/settings.json b/meilisearch-http/tests/assets/dumps/v1/test/settings.json new file mode 100644 index 000000000..918cfab53 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/settings.json @@ -0,0 +1,59 @@ +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": "email", + "searchableAttributes": [ + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "displayedAttributes": [ + "id", + "isActive", + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "stopWords": [ + "in", + "ad" + ], + "synonyms": { + "wolverine": ["xmen", "logan"], + "logan": ["wolverine", "xmen"] + }, + "attributesForFaceting": [ + "gender", + "color", + "tags", + "isActive" + ] +} diff --git a/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl new file mode 100644 index 000000000..0dcffdce0 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl @@ -0,0 +1,2 @@ +{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}} +{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} \ No newline at end of file diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 58a8de200..710b96a9f 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -2,7 +2,7 @@ use std::time::Duration; use actix_web::http::StatusCode; use serde_json::{json, Value}; -use tokio::time::sleep; +use tokio::time::delay_for; use super::service::Service; @@ -79,7 +79,7 @@ impl Index<'_> { return response; } - sleep(Duration::from_secs(1)).await; + delay_for(Duration::from_secs(1)).await; } panic!("Timeout waiting for update id"); } diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index d7d76445c..943284736 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -1,7 +1,8 @@ -use tempdir::TempDir; +use actix_web::http::StatusCode; use byte_unit::{Byte, ByteUnit}; use serde_json::Value; -use actix_web::http::StatusCode; +use tempdir::TempDir; +use urlencoding::encode; use meilisearch_http::data::Data; use meilisearch_http::option::{Opt, IndexerOpts}; @@ -60,7 +61,7 @@ impl Server { /// Returns a view to an index. There is no guarantee that the index exists. pub fn index<'a>(&'a self, uid: impl AsRef) -> Index<'a> { Index { - uid: uid.as_ref().to_string(), + uid: encode(uid.as_ref()), service: &self.service, } } diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index feff49cad..8e797c00d 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -2,14 +2,14 @@ use actix_web::{http::StatusCode, test}; use serde_json::Value; use meilisearch_http::data::Data; -use meilisearch_http::create_app; +use meilisearch_http::helpers::NormalizePath; pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -26,12 +26,12 @@ impl Service { /// Send a test post request from a text body, with a `content-type:application/json` header. pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) - .insert_header(("content-type", "application/json")) + .header("content-type", "application/json") .to_request(); let res = test::call_service(&mut app, req).await; let status_code = res.status(); @@ -43,7 +43,7 @@ impl Service { pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; @@ -56,7 +56,7 @@ impl Service { pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::put() .uri(url.as_ref()) @@ -72,7 +72,7 @@ impl Service { pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 70d6aab68..63724af18 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -45,6 +45,22 @@ async fn add_documents_no_index_creation() { assert_eq!(response["primaryKey"], "id"); } +#[actix_rt::test] +async fn document_add_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (_response, code) = index.add_documents(json!([]), None).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn document_update_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (_response, code) = index.update_documents(json!([]), None).await; + assert_eq!(code, 400); +} + #[actix_rt::test] async fn document_addition_with_primary_key() { let server = Server::new().await; diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 3ff452c33..c26941b91 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -47,12 +47,10 @@ async fn create_existing_index() { assert_eq!(code, 400); } -// test fails (issue #46) #[actix_rt::test] -#[ignore] async fn create_with_invalid_index_uid() { let server = Server::new().await; - let index = server.index("test test"); + let index = server.index("test test#!"); let (_, code) = index.create(None).await; assert_eq!(code, 400); } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index bae044acb..e5fbbde35 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -16,10 +16,11 @@ async fn get_settings() { let (response, code) = index.settings().await; assert_eq!(code, 200); let settings = response.as_object().unwrap(); - assert_eq!(settings.keys().len(), 3); + assert_eq!(settings.keys().len(), 4); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["facetedAttributes"], json!({})); + assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximity", "attribute", "wordsPosition", "exactness"])); } #[actix_rt::test] @@ -51,8 +52,6 @@ async fn test_partial_update() { } #[actix_rt::test] -#[ignore] -// need fix #54 async fn delete_settings_unexisting_index() { let server = Server::new().await; let index = server.index("test"); @@ -90,6 +89,15 @@ async fn update_setting_unexisting_index() { assert_eq!(code, 200); } +#[actix_rt::test] +async fn update_setting_unexisting_index_invalid_uid() { + let server = Server::new().await; + let index = server.index("test##! "); + let (_response, code) = index.update_settings(json!({})).await; + println!("response: {}", _response); + assert_eq!(code, 400); +} + macro_rules! test_setting_routes { ($($setting:ident), *) => { $( @@ -123,7 +131,6 @@ macro_rules! test_setting_routes { } #[actix_rt::test] - #[ignore] async fn delete_unexisting_index() { let server = Server::new().await; let url = format!("/indexes/test/settings/{}", diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 3fff2d911..03b307daf 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -50,9 +50,7 @@ async fn list_no_updates() { assert!(response.as_array().unwrap().is_empty()); } -// TODO: fix #32 #[actix_rt::test] -#[ignore] async fn list_updates() { let server = Server::new().await; let index = server.index("test"); From a56e8c1a0c3d46b9787235f6f81045f682a9cf60 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 14:47:04 +0100 Subject: [PATCH 152/522] fix tests --- Cargo.lock | 7 +++++++ meilisearch-http/Cargo.toml | 1 + meilisearch-http/tests/common/index.rs | 4 ++-- meilisearch-http/tests/common/service.rs | 14 +++++++------- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f8724ca1..84c9b6854 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1817,6 +1817,7 @@ dependencies = [ "tempfile", "thiserror", "tokio 1.3.0", + "urlencoding", "uuid", "vergen", ] @@ -3503,6 +3504,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" + [[package]] name = "utf8-width" version = "0.1.4" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 0f3d3cb38..b09c52b87 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -74,6 +74,7 @@ serde_url_params = "0.2.0" tempdir = "0.3.7" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } actix-rt = "2.1.0" +urlencoding = "1.1.1" [features] default = ["sentry"] diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 710b96a9f..58a8de200 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -2,7 +2,7 @@ use std::time::Duration; use actix_web::http::StatusCode; use serde_json::{json, Value}; -use tokio::time::delay_for; +use tokio::time::sleep; use super::service::Service; @@ -79,7 +79,7 @@ impl Index<'_> { return response; } - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; } panic!("Timeout waiting for update id"); } diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index 8e797c00d..feff49cad 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -2,14 +2,14 @@ use actix_web::{http::StatusCode, test}; use serde_json::Value; use meilisearch_http::data::Data; -use meilisearch_http::helpers::NormalizePath; +use meilisearch_http::create_app; pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -26,12 +26,12 @@ impl Service { /// Send a test post request from a text body, with a `content-type:application/json` header. pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) - .header("content-type", "application/json") + .insert_header(("content-type", "application/json")) .to_request(); let res = test::call_service(&mut app, req).await; let status_code = res.status(); @@ -43,7 +43,7 @@ impl Service { pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; @@ -56,7 +56,7 @@ impl Service { pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::put() .uri(url.as_ref()) @@ -72,7 +72,7 @@ impl Service { pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; From 53cf500e36f0e477a18133f5335d3870ff2b2799 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 18:04:20 +0100 Subject: [PATCH 153/522] uuid resolver hard state --- meilisearch-http/src/data/updates.rs | 3 - .../local_index_controller/mod.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 2 +- .../src/index_controller/uuid_resolver.rs | 193 ++++++++++++++---- meilisearch-http/src/routes/index.rs | 2 +- meilisearch-http/tests/common/index.rs | 6 +- meilisearch-http/tests/updates/mod.rs | 2 +- 7 files changed, 161 insertions(+), 49 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index e9d1b51b7..c6e30ea02 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,7 +1,4 @@ -//use async_compression::tokio_02::write::GzipEncoder; -//use futures_util::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; -//use tokio::io::AsyncWriteExt; use actix_web::web::Payload; use crate::index_controller::{UpdateStatus, IndexMetadata}; diff --git a/meilisearch-http/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs index 14efe42c7..8ac600f5f 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/mod.rs @@ -148,7 +148,7 @@ impl IndexController for LocalIndexController { fn update_index(&self, uid: impl AsRef, index_settings: IndexSettings) -> anyhow::Result { if index_settings.name.is_some() { - bail!("can't udpate an index name.") + bail!("can't update an index name.") } let (primary_key, meta) = match index_settings.primary_key { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index a824852bf..85469728b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -69,7 +69,7 @@ enum IndexControllerMsg { impl IndexController { pub fn new(path: impl AsRef) -> anyhow::Result { - let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; let index_actor = index_actor::IndexActorHandle::new(&path)?; let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 8d33edef4..a369790f3 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -1,9 +1,9 @@ +use std::{fs::create_dir_all, path::Path}; + +use heed::{Database, Env, EnvOpenOptions, types::{ByteSlice, Str}}; use log::{info, warn}; -use std::collections::hash_map::Entry; -use std::collections::HashMap; -use std::sync::Arc; use thiserror::Error; -use tokio::sync::{mpsc, oneshot, RwLock}; +use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; pub type Result = std::result::Result; @@ -81,14 +81,14 @@ impl UuidResolverActor { async fn handle_resolve(&self, name: String) -> Result { self.store - .get_uuid(&name) + .get_uuid(name.clone()) .await? .ok_or(UuidError::UnexistingIndex(name)) } async fn handle_delete(&self, name: String) -> Result { self.store - .delete(&name) + .delete(name.clone()) .await? .ok_or(UuidError::UnexistingIndex(name)) } @@ -105,12 +105,12 @@ pub struct UuidResolverHandle { } impl UuidResolverHandle { - pub fn new() -> Self { + pub fn new(path: impl AsRef) -> anyhow::Result { let (sender, reveiver) = mpsc::channel(100); - let store = MapUuidStore(Arc::new(RwLock::new(HashMap::new()))); + let store = HeedUuidStore::new(path)?; let actor = UuidResolverActor::new(reveiver, store); tokio::spawn(actor.run()); - Self { sender } + Ok(Self { sender }) } pub async fn resolve(&self, name: String) -> anyhow::Result { @@ -159,12 +159,18 @@ impl UuidResolverHandle { } } -#[derive(Clone, Debug, Error)] +#[derive(Debug, Error)] pub enum UuidError { #[error("Name already exist.")] NameAlreadyExist, #[error("Index \"{0}\" doesn't exist.")] UnexistingIndex(String), + #[error("Error performing task: {0}")] + TokioTask(#[from] tokio::task::JoinError), + #[error("Database error: {0}")] + Heed(#[from] heed::Error), + #[error("Uuid error: {0}")] + Uuid(#[from] uuid::Error), } #[async_trait::async_trait] @@ -172,48 +178,157 @@ trait UuidStore { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return // the uuid otherwise. async fn create_uuid(&self, name: String, err: bool) -> Result; - async fn get_uuid(&self, name: &str) -> Result>; - async fn delete(&self, name: &str) -> Result>; + async fn get_uuid(&self, name: String) -> Result>; + async fn delete(&self, name: String) -> Result>; async fn list(&self) -> Result>; } -struct MapUuidStore(Arc>>); +struct HeedUuidStore { + env: Env, + db: Database, +} + +fn open_or_create_database(env: &Env, name: Option<&str>) -> heed::Result> { + match env.open_database(name)? { + Some(db) => Ok(db), + None => env.create_database(name), + } +} + +impl HeedUuidStore { + fn new(path: impl AsRef) -> anyhow::Result { + let path = path.as_ref().join("index_uuids"); + create_dir_all(&path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(1_073_741_824); // 1GB + let env = options.open(path)?; + let db = open_or_create_database(&env, None)?; + Ok(Self { env, db }) + } +} #[async_trait::async_trait] -impl UuidStore for MapUuidStore { +impl UuidStore for HeedUuidStore { async fn create_uuid(&self, name: String, err: bool) -> Result { - match self.0.write().await.entry(name) { - Entry::Occupied(entry) => { - if err { - Err(UuidError::NameAlreadyExist) - } else { - Ok(entry.get().clone()) + let env = self.env.clone(); + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let mut txn = env.write_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + if err { + Err(UuidError::NameAlreadyExist) + } else { + let uuid = Uuid::from_slice(uuid)?; + Ok(uuid) + } + } + None => { + let uuid = Uuid::new_v4(); + db.put(&mut txn, &name, uuid.as_bytes())?; + txn.commit()?; + Ok(uuid) } } - Entry::Vacant(entry) => { - let uuid = Uuid::new_v4(); - let uuid = entry.insert(uuid); - Ok(uuid.clone()) + }).await? + } + + async fn get_uuid(&self, name: String) -> Result> { + let env = self.env.clone(); + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = env.read_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + Ok(Some(uuid)) + } + None => Ok(None), } - } + }).await? } - async fn get_uuid(&self, name: &str) -> Result> { - Ok(self.0.read().await.get(name).cloned()) - } - - async fn delete(&self, name: &str) -> Result> { - Ok(self.0.write().await.remove(name)) + async fn delete(&self, name: String) -> Result> { + let env = self.env.clone(); + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let mut txn = env.write_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + db.delete(&mut txn, &name)?; + txn.commit()?; + Ok(None) + } + None => Ok(None) + } + }).await? } async fn list(&self) -> Result> { - let list = self - .0 - .read() - .await - .iter() - .map(|(name, uuid)| (name.to_owned(), uuid.clone())) - .collect(); - Ok(list) + let env = self.env.clone(); + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = env.read_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (name, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push((name.to_owned(), uuid)) + } + Ok(entries) + }).await? + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + use std::collections::hash_map::Entry; + use std::sync::Arc; + + use tokio::sync::RwLock; + + use super::*; + + struct MapUuidStore(Arc>>); + + #[async_trait::async_trait] + impl UuidStore for MapUuidStore { + async fn create_uuid(&self, name: String, err: bool) -> Result { + match self.0.write().await.entry(name) { + Entry::Occupied(entry) => { + if err { + Err(UuidError::NameAlreadyExist) + } else { + Ok(entry.get().clone()) + } + } + Entry::Vacant(entry) => { + let uuid = Uuid::new_v4(); + let uuid = entry.insert(uuid); + Ok(uuid.clone()) + } + } + } + + async fn get_uuid(&self, name: String) -> Result> { + Ok(self.0.read().await.get(&name).cloned()) + } + + async fn delete(&self, name: String) -> Result> { + Ok(self.0.write().await.remove(&name)) + } + + async fn list(&self) -> Result> { + let list = self + .0 + .read() + .await + .iter() + .map(|(name, uuid)| (name.to_owned(), uuid.clone())) + .collect(); + Ok(list) + } } } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 818c87303..5c6e3f5a9 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -140,7 +140,7 @@ async fn get_update_status( Ok(HttpResponse::Ok().body(json)) } Ok(None) => { - let e = format!("udpate {} for index {:?} doesn't exists.", path.update_id, path.index_uid); + let e = format!("update {} for index {:?} doesn't exists.", path.update_id, path.index_uid); Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } Err(e) => { diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 58a8de200..a14769ee1 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -73,7 +73,7 @@ impl Index<'_> { let url = format!("/indexes/{}/updates/{}", self.uid, update_id); for _ in 0..10 { let (response, status_code) = self.service.get(&url).await; - assert_eq!(status_code, 200); + assert_eq!(status_code, 200, "response: {}", response); if response["status"] == "processed" || response["status"] == "failed" { return response; @@ -84,8 +84,8 @@ impl Index<'_> { panic!("Timeout waiting for update id"); } - pub async fn get_update(&self, udpate_id: u64) -> (Value, StatusCode) { - let url = format!("/indexes/{}/updates/{}", self.uid, udpate_id); + pub async fn get_update(&self, update_id: u64) -> (Value, StatusCode) { + let url = format!("/indexes/{}/updates/{}", self.uid, update_id); self.service.get(url).await } diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 03b307daf..64b5b560e 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -8,7 +8,7 @@ async fn get_update_unexisting_index() { } #[actix_rt::test] -async fn get_unexisting_udpate_status() { +async fn get_unexisting_update_status() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; From 2ae90f9c5d64cdac3847ef0044e18c1261db8ab8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 14:23:11 +0100 Subject: [PATCH 154/522] lazy load update store --- .../src/index_controller/update_actor.rs | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 370912dcf..32eda7bcb 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -301,13 +301,33 @@ impl UpdateStoreStore for MapUpdateStoreStore { } async fn get(&self, uuid: &Uuid) -> Result>> { - Ok(self.db.read().await.get(uuid).cloned()) + // attemps to get pre-loaded ref to the index + match self.db.read().await.get(uuid) { + Some(uuid) => Ok(Some(uuid.clone())), + None => { + // otherwise we try to check if it exists, and load it. + let path = self.path.clone().join(format!("updates-{}", uuid)); + if path.exists() { + let index_handle = self.index_handle.clone(); + let mut options = heed::EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle.update(meta, file)) + }) + .unwrap(); + self.db.write().await.insert(uuid.clone(), store.clone()); + Ok(Some(store)) + } else { + Ok(None) + } + } + } } async fn delete(&self, uuid: &Uuid) -> Result>> { let store = self.db.write().await.remove(&uuid); - if store.is_some() { - let path = self.path.clone().join(format!("updates-{}", uuid)); + let path = self.path.clone().join(format!("updates-{}", uuid)); + if store.is_some() || path.exists() { remove_dir_all(path).unwrap(); } Ok(store) From 1fad72e019ef09279571335edd677cb8453f3e28 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 17:59:47 +0100 Subject: [PATCH 155/522] fix test bug with tempdir --- .../src/index_controller/index_actor.rs | 316 ++++++++++-------- .../local_index_controller/index_store.rs | 3 +- meilisearch-http/src/index_controller/mod.rs | 2 +- .../src/index_controller/update_actor.rs | 13 +- .../src/index_controller/uuid_resolver.rs | 3 +- meilisearch-http/tests/common/server.rs | 9 +- .../tests/settings/get_settings.rs | 3 +- 7 files changed, 188 insertions(+), 161 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 1ddc041a1..d8dc5f014 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -1,4 +1,4 @@ -use std::collections::{hash_map::Entry, HashMap}; +use std::collections::HashMap; use std::fs::{create_dir_all, remove_dir_all, File}; use std::future::Future; use std::path::{Path, PathBuf}; @@ -8,11 +8,15 @@ use async_stream::stream; use chrono::{DateTime, Utc}; use futures::pin_mut; use futures::stream::StreamExt; -use heed::EnvOpenOptions; -use log::info; +use heed::{ + types::{ByteSlice, SerdeBincode}, + Database, Env, EnvOpenOptions, +}; +use log::debug; use serde::{Deserialize, Serialize}; use thiserror::Error; -use tokio::sync::{mpsc, oneshot, RwLock}; +use tokio::{sync::{mpsc, oneshot, RwLock}}; +use tokio::task::spawn_blocking; use uuid::Uuid; use super::get_arc_ownership_blocking; @@ -96,6 +100,8 @@ pub enum IndexError { IndexAlreadyExists, #[error("Index doesn't exists")] UnexistingIndex, + #[error("Heed error: {0}")] + HeedError(#[from] heed::Error), } #[async_trait::async_trait] @@ -105,10 +111,9 @@ trait IndexStore { where F: FnOnce(Index) -> Result + Send + Sync + 'static, R: Sync + Send + 'static; - async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; - async fn delete(&self, uuid: &Uuid) -> Result>; - async fn get_meta(&self, uuid: &Uuid) -> Result>; + async fn delete(&self, uuid: Uuid) -> Result>; + async fn get_meta(&self, uuid: Uuid) -> Result>; } impl IndexActor { @@ -118,8 +123,8 @@ impl IndexActor { store: S, ) -> Result { let options = IndexerOpts::default(); - let update_handler = UpdateHandler::new(&options) - .map_err(|e| IndexError::Error(e.into()))?; + let update_handler = + UpdateHandler::new(&options).map_err(|e| IndexError::Error(e.into()))?; let update_handler = Arc::new(update_handler); let read_receiver = Some(read_receiver); let write_receiver = Some(write_receiver); @@ -227,11 +232,12 @@ impl IndexActor { } async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> anyhow::Result { - let index = self.store + let index = self + .store .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - tokio::task::spawn_blocking(move || index.perform_search(query)).await? + spawn_blocking(move || index.perform_search(query)).await? } async fn handle_create_index( @@ -247,15 +253,14 @@ impl IndexActor { meta: Processing, data: File, ) -> Result { - info!("Processing update {}", meta.id()); + debug!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let update_handler = self.update_handler.clone(); let handle = self .store .update_index(uuid, |index| { - let handle = tokio::task::spawn_blocking(move || { - update_handler.handle_update(meta, data, index) - }); + let handle = + spawn_blocking(move || update_handler.handle_update(meta, data, index)); Ok(handle) }) .await?; @@ -264,11 +269,12 @@ impl IndexActor { } async fn handle_settings(&self, uuid: Uuid) -> Result { - let index = self.store + let index = self + .store .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - tokio::task::spawn_blocking(move || index.settings().map_err(|e| IndexError::Error(e))) + spawn_blocking(move || index.settings().map_err(|e| IndexError::Error(e))) .await .map_err(|e| IndexError::Error(e.into()))? } @@ -280,10 +286,12 @@ impl IndexActor { limit: usize, attributes_to_retrieve: Option>, ) -> Result> { - let index = self.store.get(uuid) + let index = self + .store + .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - tokio::task::spawn_blocking(move || { + spawn_blocking(move || { index .retrieve_documents(offset, limit, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)) @@ -303,7 +311,7 @@ impl IndexActor { .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - tokio::task::spawn_blocking(move || { + spawn_blocking(move || { index .retrieve_document(doc_id, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)) @@ -313,15 +321,15 @@ impl IndexActor { } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let index = self.store.delete(&uuid).await?; + let index = self.store.delete(uuid).await?; if let Some(index) = index { tokio::task::spawn(async move { let index = index.0; let store = get_arc_ownership_blocking(index).await; - tokio::task::spawn_blocking(move || { + spawn_blocking(move || { store.prepare_for_closing().wait(); - info!("Index {} was closed.", uuid); + debug!("Index closed"); }); }); } @@ -330,7 +338,7 @@ impl IndexActor { } async fn handle_get_meta(&self, uuid: Uuid) -> Result> { - let result = self.store.get_meta(&uuid).await?; + let result = self.store.get_meta(uuid).await?; Ok(result) } } @@ -346,7 +354,7 @@ impl IndexActorHandle { let (read_sender, read_receiver) = mpsc::channel(100); let (write_sender, write_receiver) = mpsc::channel(100); - let store = MapIndexStore::new(path); + let store = HeedIndexStore::new(path)?; let actor = IndexActor::new(read_receiver, write_receiver, store)?; tokio::task::spawn(actor.run()); Ok(Self { @@ -442,149 +450,165 @@ impl IndexActorHandle { } } -struct MapIndexStore { - root: PathBuf, - meta_store: AsyncMap, +struct HeedIndexStore { + env: Env, + db: Database>, index_store: AsyncMap, + path: PathBuf, +} + +impl HeedIndexStore { + fn new(path: impl AsRef) -> anyhow::Result { + let mut options = EnvOpenOptions::new(); + options.map_size(1_073_741_824); //1GB + let path = path.as_ref().join("indexes/"); + create_dir_all(&path)?; + let env = options.open(&path)?; + let db = env.create_database(None)?; + let index_store = Arc::new(RwLock::new(HashMap::new())); + Ok(Self { + env, + db, + index_store, + path, + }) + } } #[async_trait::async_trait] -impl IndexStore for MapIndexStore { +impl IndexStore for HeedIndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { - let meta = match self.meta_store.write().await.entry(uuid.clone()) { - Entry::Vacant(entry) => { - let now = Utc::now(); - let meta = IndexMeta { - uuid, - created_at: now.clone(), - updated_at: now, - primary_key, - }; - entry.insert(meta).clone() - } - Entry::Occupied(_) => return Err(IndexError::IndexAlreadyExists), - }; + let path = self.path.join(format!("index-{}", uuid)); - let db_path = self.root.join(format!("index-{}", meta.uuid)); + if path.exists() { + return Err(IndexError::IndexAlreadyExists); + } - let index: Result = tokio::task::spawn_blocking(move || { - create_dir_all(&db_path).expect("can't create db"); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; - let index = Index(Arc::new(index)); - Ok(index) + let env = self.env.clone(); + let db = self.db.clone(); + let (index, meta) = spawn_blocking(move || -> Result<(Index, IndexMeta)> { + let now = Utc::now(); + let meta = IndexMeta { + uuid: uuid.clone(), + created_at: now.clone(), + updated_at: now, + primary_key, + }; + let mut txn = env.write_txn()?; + db.put(&mut txn, uuid.as_bytes(), &meta)?; + txn.commit()?; + + let index = open_index(&path, 4096 * 100_000)?; + + Ok((index, meta)) }) .await - .expect("thread died"); + .expect("thread died")?; - self.index_store - .write() - .await - .insert(meta.uuid.clone(), index?); + self.index_store.write().await.insert(uuid.clone(), index); Ok(meta) } - async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result { - match self.index_store.write().await.entry(uuid.clone()) { - Entry::Vacant(index_entry) => match self.meta_store.write().await.entry(uuid.clone()) { - Entry::Vacant(meta_entry) => { - let now = Utc::now(); - let meta = IndexMeta { - uuid, - created_at: now.clone(), - updated_at: now, - primary_key, - }; - let meta = meta_entry.insert(meta); - let db_path = self.root.join(format!("index-{}", meta.uuid)); - - let index: Result = tokio::task::spawn_blocking(move || { - create_dir_all(&db_path).expect("can't create db"); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path) - .map_err(|e| IndexError::Error(e))?; - let index = Index(Arc::new(index)); - Ok(index) - }) - .await - .expect("thread died"); - - Ok(index_entry.insert(index?).clone()) - } - Entry::Occupied(entry) => { - let meta = entry.get(); - let db_path = self.root.join(format!("index-{}", meta.uuid)); - - let index: Result = tokio::task::spawn_blocking(move || { - create_dir_all(&db_path).expect("can't create db"); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path) - .map_err(|e| IndexError::Error(e))?; - let index = Index(Arc::new(index)); - Ok(index) - }) - .await - .expect("thread died"); - - Ok(index_entry.insert(index?).clone()) - } - }, - Entry::Occupied(entry) => Ok(entry.get().clone()), - } - } - - async fn get(&self, uuid: Uuid) -> Result> { - Ok(self.index_store.read().await.get(&uuid).cloned()) - } - - async fn delete(&self, uuid: &Uuid) -> Result> { - let index = self.index_store.write().await.remove(&uuid); - if index.is_some() { - let db_path = self.root.join(format!("index-{}", uuid)); - remove_dir_all(db_path).unwrap(); - } - Ok(index) - } - - async fn get_meta(&self, uuid: &Uuid) -> Result> { - Ok(self.meta_store.read().await.get(uuid).cloned()) - } - async fn update_index(&self, uuid: Uuid, f: F) -> Result where F: FnOnce(Index) -> Result + Send + Sync + 'static, R: Sync + Send + 'static, { - let index = self.get_or_create(uuid.clone(), None).await?; - let mut meta = self - .get_meta(&uuid) - .await? - .ok_or(IndexError::UnexistingIndex)?; - match f(index) { - Ok(r) => { - meta.updated_at = Utc::now(); - self.meta_store.write().await.insert(uuid, meta); - Ok(r) + let guard = self.index_store.read().await; + let index = match guard.get(&uuid) { + Some(index) => index.clone(), + None => { + drop(guard); + self.create_index(uuid.clone(), None).await?; + self.index_store + .read() + .await + .get(&uuid) + .expect("Index should exist") + .clone() + } + }; + + let env = self.env.clone(); + let db = self.db.clone(); + spawn_blocking(move || { + let mut txn = env.write_txn()?; + let mut meta = db.get(&txn, uuid.as_bytes())?.expect("unexisting index"); + match f(index) { + Ok(r) => { + meta.updated_at = Utc::now(); + db.put(&mut txn, uuid.as_bytes(), &meta)?; + txn.commit()?; + Ok(r) + } + Err(e) => Err(e), + } + }) + .await + .expect("thread died") + } + + async fn get(&self, uuid: Uuid) -> Result> { + let guard = self.index_store.read().await; + match guard.get(&uuid) { + Some(index) => Ok(Some(index.clone())), + None => { + // drop the guard here so we can perform the write after without deadlocking; + drop(guard); + let path = self.path.join(format!("index-{}", uuid)); + if !path.exists() { + return Ok(None); + } + + // TODO: set this info from the database + let index = spawn_blocking(|| open_index(path, 4096 * 100_000)) + .await + .expect("thread died")?; + self.index_store + .write() + .await + .insert(uuid.clone(), index.clone()); + println!("here"); + Ok(Some(index)) } - Err(e) => Err(e), } } + + async fn delete(&self, uuid: Uuid) -> Result> { + let env = self.env.clone(); + let db = self.db.clone(); + let db_path = self.path.join(format!("index-{}", uuid)); + spawn_blocking(move || -> Result<()> { + let mut txn = env.write_txn()?; + db.delete(&mut txn, uuid.as_bytes())?; + txn.commit()?; + remove_dir_all(db_path).unwrap(); + Ok(()) + }) + .await + .expect("thread died")?; + let index = self.index_store.write().await.remove(&uuid); + Ok(index) + } + + async fn get_meta(&self, uuid: Uuid) -> Result> { + let env = self.env.clone(); + let db = self.db.clone(); + spawn_blocking(move || { + let txn = env.read_txn()?; + let meta = db.get(&txn, uuid.as_bytes())?; + Ok(meta) + }) + .await + .expect("thread died") + } } -impl MapIndexStore { - fn new(root: impl AsRef) -> Self { - let mut root = root.as_ref().to_owned(); - root.push("indexes/"); - let meta_store = Arc::new(RwLock::new(HashMap::new())); - let index_store = Arc::new(RwLock::new(HashMap::new())); - Self { - meta_store, - index_store, - root, - } - } +fn open_index(path: impl AsRef, size: usize) -> Result { + create_dir_all(&path).expect("can't create db"); + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, &path).map_err(|e| IndexError::Error(e))?; + Ok(Index(Arc::new(index))) } diff --git a/meilisearch-http/src/index_controller/local_index_controller/index_store.rs b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs index 3fe8a3f59..a690abaf4 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/index_store.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs @@ -12,8 +12,7 @@ use heed::{ }; use log::{error, info}; use milli::Index; -use rayon::ThreadPool; -use serde::{Deserialize, Serialize}; +use rayon::ThreadPool; use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::update_handler::UpdateHandler; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 85469728b..2c181817b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -71,7 +71,7 @@ impl IndexController { pub fn new(path: impl AsRef) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; let index_actor = index_actor::IndexActorHandle::new(&path)?; - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path)?; Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 32eda7bcb..708e8fd19 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -68,10 +68,11 @@ where D: AsRef<[u8]> + Sized + 'static, S: UpdateStoreStore, { - fn new(store: S, inbox: mpsc::Receiver>, path: impl AsRef) -> Self { + fn new(store: S, inbox: mpsc::Receiver>, path: impl AsRef) -> anyhow::Result { let path = path.as_ref().to_owned().join("update_files"); - create_dir_all(&path).unwrap(); - Self { store, inbox, path } + create_dir_all(&path)?; + assert!(path.exists()); + Ok(Self { store, inbox, path }) } async fn run(mut self) { @@ -211,15 +212,15 @@ impl UpdateActorHandle where D: AsRef<[u8]> + Sized + 'static + Sync + Send, { - pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { + pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> anyhow::Result { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); let store = MapUpdateStoreStore::new(index_handle, &path); - let actor = UpdateActor::new(store, receiver, path); + let actor = UpdateActor::new(store, receiver, path)?; tokio::task::spawn(actor.run()); - Self { sender } + Ok(Self { sender }) } pub async fn update( diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index a369790f3..d8d39d922 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -277,8 +277,7 @@ impl UuidStore for HeedUuidStore { entries.push((name.to_owned(), uuid)) } Ok(entries) - }).await? - } + }).await? } } #[cfg(test)] diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 943284736..090524601 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -12,15 +12,17 @@ use super::service::Service; pub struct Server { pub service: Service, + // hod ownership to the tempdir while we use the server instance. + _dir: tempdir::TempDir, } impl Server { pub async fn new() -> Self { - let tmp_dir = TempDir::new("meilisearch").unwrap(); + let dir = TempDir::new("meilisearch").unwrap(); let opt = Opt { - db_path: tmp_dir.path().join("db"), - dumps_dir: tmp_dir.path().join("dump"), + db_path: dir.path().join("db"), + dumps_dir: dir.path().join("dump"), dump_batch_size: 16, http_addr: "127.0.0.1:7700".to_owned(), master_key: None, @@ -55,6 +57,7 @@ impl Server { Server { service, + _dir: dir, } } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index e5fbbde35..ba5d9651c 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -35,7 +35,8 @@ async fn update_settings_unknown_field() { async fn test_partial_update() { let server = Server::new().await; let index = server.index("test"); - index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + let (response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + println!("response: {}", response); index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); From 79a4bc8129349d63ed505a1e4a7a79aab8b2828a Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 19:40:18 +0100 Subject: [PATCH 156/522] use meta from milli --- Cargo.lock | 228 ++++++++++++------ meilisearch-http/Cargo.toml | 3 +- meilisearch-http/src/index/search.rs | 12 +- meilisearch-http/src/index/updates.rs | 2 +- .../src/index_controller/index_actor.rs | 172 ++++--------- 5 files changed, 207 insertions(+), 210 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84c9b6854..575f03d98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,8 +159,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" dependencies = [ - "quote", - "syn", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -169,8 +169,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ - "quote", - "syn", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -363,9 +363,9 @@ version = "0.5.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -462,9 +462,9 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -473,9 +473,9 @@ version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -925,9 +925,9 @@ version = "0.99.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -982,9 +982,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" dependencies = [ "heck", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -1029,9 +1029,9 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "synstructure", ] @@ -1170,9 +1170,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -1721,6 +1721,28 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "logging_timer" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d0c249955c17c2f8f86b5f501b16d2509ebbe775f7b1d1d2b1ba85ade2a793" +dependencies = [ + "log", + "logging_timer_proc_macros", +] + +[[package]] +name = "logging_timer_proc_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482c2c28e6bcfe7c4274f82f701774d755e6aa873edfd619460fcd0966e0eb07" +dependencies = [ + "log", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -1866,11 +1888,11 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" dependencies = [ "anyhow", "bstr", "byteorder", + "chrono", "crossbeam-channel", "csv", "either", @@ -1884,6 +1906,7 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", + "logging_timer", "meilisearch-tokenizer", "memmap", "num-traits", @@ -1897,9 +1920,11 @@ dependencies = [ "roaring", "serde", "serde_json", + "slice-group-by", "smallstr", "smallvec", "tempfile", + "tinytemplate", "uuid", ] @@ -2170,9 +2195,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "pest_meta", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2248,9 +2273,9 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2259,9 +2284,9 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2301,9 +2326,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "version_check", ] @@ -2313,8 +2338,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "version_check", ] @@ -2330,13 +2355,22 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ - "unicode-xid", + "unicode-xid 0.2.1", ] [[package]] @@ -2345,13 +2379,22 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.24", ] [[package]] @@ -2799,9 +2842,9 @@ version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2987,11 +3030,11 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "serde", "serde_derive", - "syn", + "syn 1.0.63", ] [[package]] @@ -3001,13 +3044,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "serde", "serde_derive", "serde_json", "sha1", - "syn", + "syn 1.0.63", ] [[package]] @@ -3041,9 +3084,20 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", +] + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", ] [[package]] @@ -3052,9 +3106,9 @@ version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.9", + "unicode-xid 0.2.1", ] [[package]] @@ -3072,10 +3126,10 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", + "unicode-xid 0.2.1", ] [[package]] @@ -3146,9 +3200,9 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -3212,10 +3266,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "standback", - "syn", + "syn 1.0.63", +] + +[[package]] +name = "tinytemplate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" +dependencies = [ + "serde", + "serde_json", ] [[package]] @@ -3281,9 +3345,9 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -3479,6 +3543,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.1" @@ -3592,9 +3662,9 @@ dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "wasm-bindgen-shared", ] @@ -3616,7 +3686,7 @@ version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" dependencies = [ - "quote", + "quote 1.0.9", "wasm-bindgen-macro-support", ] @@ -3626,9 +3696,9 @@ version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3788,8 +3858,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ - "proc-macro2", - "syn", + "proc-macro2 1.0.24", + "syn 1.0.63", "synstructure", ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index b09c52b87..15319dea6 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -38,7 +38,8 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } +milli = { path = "../../milli/milli" } +#milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 1264d49d6..fb830ec60 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -6,7 +6,7 @@ use either::Either; use anyhow::bail; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{FacetCondition, facet::FacetValue}; +use milli::{FacetCondition, MatchingWords, facet::FacetValue}; use serde::{Serialize, Deserialize}; use serde_json::{Value, Map}; @@ -71,7 +71,7 @@ impl Index { let milli::SearchResult { documents_ids, - found_words, + matching_words, candidates, .. } = search.execute()?; @@ -102,7 +102,7 @@ impl Index { for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } documents.push(object); } @@ -173,7 +173,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { Self { analyzer } } - pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + pub fn highlight_value(&self, value: Value, words_to_highlight: &MatchingWords) -> Value { match value { Value::Null => Value::Null, Value::Bool(boolean) => Value::Bool(boolean), @@ -183,7 +183,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { let analyzed = self.analyzer.analyze(&old_string); for (word, token) in analyzed.reconstruct() { if token.is_word() { - let to_highlight = words_to_highlight.contains(token.text()); + let to_highlight = words_to_highlight.matches(token.text()); if to_highlight { string.push_str("") } @@ -215,7 +215,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { pub fn highlight_record( &self, object: &mut Map, - words_to_highlight: &HashSet, + words_to_highlight: &MatchingWords, attributes_to_highlight: &HashSet, ) { // TODO do we need to create a string for element that are not and needs to be highlight? diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index d339406f7..129a340a7 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -12,7 +12,7 @@ use super::Index; #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { DocumentsAddition(DocumentAdditionResult), - DocumentDeletion { deleted: usize }, + DocumentDeletion { deleted: u64 }, Other, } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index d8dc5f014..b5d5792b2 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::fs::{create_dir_all, remove_dir_all, File}; +use std::fs::{create_dir_all, File}; use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -8,15 +8,13 @@ use async_stream::stream; use chrono::{DateTime, Utc}; use futures::pin_mut; use futures::stream::StreamExt; -use heed::{ - types::{ByteSlice, SerdeBincode}, - Database, Env, EnvOpenOptions, -}; +use heed::EnvOpenOptions; use log::debug; use serde::{Deserialize, Serialize}; use thiserror::Error; -use tokio::{sync::{mpsc, oneshot, RwLock}}; +use tokio::sync::{mpsc, oneshot, RwLock}; use tokio::task::spawn_blocking; +use tokio::fs::remove_dir_all; use uuid::Uuid; use super::get_arc_ownership_blocking; @@ -36,12 +34,21 @@ type UpdateResult = std::result::Result, Failed, updated_at: DateTime, primary_key: Option, } +impl IndexMeta { + fn new(index: &Index) -> Result { + let txn = index.read_txn()?; + let created_at = index.created_at(&txn)?; + let updated_at = index.updated_at(&txn)?; + let primary_key = index.primary_key(&txn)?.map(String::from); + Ok(Self { primary_key, updated_at, created_at }) + } +} + enum IndexMsg { CreateIndex { uuid: Uuid, @@ -106,14 +113,9 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; - async fn update_index(&self, uuid: Uuid, f: F) -> Result - where - F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static; + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: Uuid) -> Result>; - async fn get_meta(&self, uuid: Uuid) -> Result>; } impl IndexActor { @@ -245,7 +247,11 @@ impl IndexActor { uuid: Uuid, primary_key: Option, ) -> Result { - self.store.create_index(uuid, primary_key).await + let index = self.store.create(uuid, primary_key).await?; + let meta = spawn_blocking(move || IndexMeta::new(&index)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + Ok(meta) } async fn handle_update( @@ -256,16 +262,13 @@ impl IndexActor { debug!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let update_handler = self.update_handler.clone(); - let handle = self - .store - .update_index(uuid, |index| { - let handle = - spawn_blocking(move || update_handler.handle_update(meta, data, index)); - Ok(handle) - }) - .await?; - - handle.await.map_err(|e| IndexError::Error(e.into())) + let index = match self.store.get(uuid.clone()).await? { + Some(index) => index, + None => self.store.create(uuid, None).await?, + }; + spawn_blocking(move || update_handler.handle_update(meta, data, index)) + .await + .map_err(|e| IndexError::Error(e.into())) } async fn handle_settings(&self, uuid: Uuid) -> Result { @@ -338,8 +341,15 @@ impl IndexActor { } async fn handle_get_meta(&self, uuid: Uuid) -> Result> { - let result = self.store.get_meta(uuid).await?; - Ok(result) + match self.store.get(uuid).await? { + Some(index) => { + let meta = spawn_blocking(move || IndexMeta::new(&index)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + Ok(Some(meta)) + } + None => Ok(None), + } } } @@ -451,24 +461,15 @@ impl IndexActorHandle { } struct HeedIndexStore { - env: Env, - db: Database>, index_store: AsyncMap, path: PathBuf, } impl HeedIndexStore { fn new(path: impl AsRef) -> anyhow::Result { - let mut options = EnvOpenOptions::new(); - options.map_size(1_073_741_824); //1GB let path = path.as_ref().join("indexes/"); - create_dir_all(&path)?; - let env = options.open(&path)?; - let db = env.create_database(None)?; let index_store = Arc::new(RwLock::new(HashMap::new())); Ok(Self { - env, - db, index_store, path, }) @@ -477,76 +478,22 @@ impl HeedIndexStore { #[async_trait::async_trait] impl IndexStore for HeedIndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result { let path = self.path.join(format!("index-{}", uuid)); - if path.exists() { return Err(IndexError::IndexAlreadyExists); } - let env = self.env.clone(); - let db = self.db.clone(); - let (index, meta) = spawn_blocking(move || -> Result<(Index, IndexMeta)> { - let now = Utc::now(); - let meta = IndexMeta { - uuid: uuid.clone(), - created_at: now.clone(), - updated_at: now, - primary_key, - }; - let mut txn = env.write_txn()?; - db.put(&mut txn, uuid.as_bytes(), &meta)?; - txn.commit()?; - + let index = spawn_blocking(move || -> Result { let index = open_index(&path, 4096 * 100_000)?; - - Ok((index, meta)) + Ok(index) }) .await - .expect("thread died")?; + .map_err(|e| IndexError::Error(e.into()))??; - self.index_store.write().await.insert(uuid.clone(), index); + self.index_store.write().await.insert(uuid.clone(), index.clone()); - Ok(meta) - } - - async fn update_index(&self, uuid: Uuid, f: F) -> Result - where - F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static, - { - let guard = self.index_store.read().await; - let index = match guard.get(&uuid) { - Some(index) => index.clone(), - None => { - drop(guard); - self.create_index(uuid.clone(), None).await?; - self.index_store - .read() - .await - .get(&uuid) - .expect("Index should exist") - .clone() - } - }; - - let env = self.env.clone(); - let db = self.db.clone(); - spawn_blocking(move || { - let mut txn = env.write_txn()?; - let mut meta = db.get(&txn, uuid.as_bytes())?.expect("unexisting index"); - match f(index) { - Ok(r) => { - meta.updated_at = Utc::now(); - db.put(&mut txn, uuid.as_bytes(), &meta)?; - txn.commit()?; - Ok(r) - } - Err(e) => Err(e), - } - }) - .await - .expect("thread died") + Ok(index) } async fn get(&self, uuid: Uuid) -> Result> { @@ -561,54 +508,33 @@ impl IndexStore for HeedIndexStore { return Ok(None); } - // TODO: set this info from the database let index = spawn_blocking(|| open_index(path, 4096 * 100_000)) .await - .expect("thread died")?; + .map_err(|e| IndexError::Error(e.into()))??; self.index_store .write() .await .insert(uuid.clone(), index.clone()); - println!("here"); Ok(Some(index)) } } } async fn delete(&self, uuid: Uuid) -> Result> { - let env = self.env.clone(); - let db = self.db.clone(); let db_path = self.path.join(format!("index-{}", uuid)); - spawn_blocking(move || -> Result<()> { - let mut txn = env.write_txn()?; - db.delete(&mut txn, uuid.as_bytes())?; - txn.commit()?; - remove_dir_all(db_path).unwrap(); - Ok(()) - }) - .await - .expect("thread died")?; + remove_dir_all(db_path).await + .map_err(|e| IndexError::Error(e.into()))?; let index = self.index_store.write().await.remove(&uuid); Ok(index) } - - async fn get_meta(&self, uuid: Uuid) -> Result> { - let env = self.env.clone(); - let db = self.db.clone(); - spawn_blocking(move || { - let txn = env.read_txn()?; - let meta = db.get(&txn, uuid.as_bytes())?; - Ok(meta) - }) - .await - .expect("thread died") - } } fn open_index(path: impl AsRef, size: usize) -> Result { - create_dir_all(&path).expect("can't create db"); + create_dir_all(&path) + .map_err(|e| IndexError::Error(e.into()))?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options, &path).map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &path) + .map_err(|e| IndexError::Error(e))?; Ok(Index(Arc::new(index))) } From 3f68460d6c3dc10e5294e013c531e2fb5ed3ad76 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 20:58:51 +0100 Subject: [PATCH 157/522] fix update dedup --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 1 + .../src/index_controller/update_actor.rs | 69 +++++++--------- .../src/index_controller/update_store.rs | 79 +++++++++++-------- 4 files changed, 79 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 575f03d98..3620a6885 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1822,6 +1822,7 @@ dependencies = [ "milli", "mime", "once_cell", + "parking_lot", "rand 0.7.3", "rayon", "regex", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 15319dea6..7b09b3042 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -62,6 +62,7 @@ either = "1.6.1" async-trait = "0.1.42" thiserror = "1.0.24" async-stream = "0.3.0" +parking_lot = "0.11.1" [dependencies.sentry] default-features = false diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 708e8fd19..8614e297b 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -3,18 +3,17 @@ use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use itertools::Itertools; -use log::info; use super::index_actor::IndexActorHandle; +use log::info; use thiserror::Error; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; +use super::get_arc_ownership_blocking; use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; -use super::get_arc_ownership_blocking; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; @@ -68,7 +67,11 @@ where D: AsRef<[u8]> + Sized + 'static, S: UpdateStoreStore, { - fn new(store: S, inbox: mpsc::Receiver>, path: impl AsRef) -> anyhow::Result { + fn new( + store: S, + inbox: mpsc::Receiver>, + path: impl AsRef, + ) -> anyhow::Result { let path = path.as_ref().to_owned().join("update_files"); create_dir_all(&path)?; assert!(path.exists()); @@ -87,12 +90,12 @@ where meta, data, ret, - }) => { + }) => { let _ = ret.send(self.handle_update(uuid, meta, data).await); } Some(ListUpdates { uuid, ret }) => { let _ = ret.send(self.handle_list_updates(uuid).await); - } , + } Some(GetUpdate { uuid, ret, id }) => { let _ = ret.send(self.handle_get_update(uuid, id).await); } @@ -113,13 +116,15 @@ where let update_store = self.store.get_or_create(uuid).await?; let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); - let mut file = File::create(&path).await + let mut file = File::create(&path) + .await .map_err(|e| UpdateError::Error(Box::new(e)))?; while let Some(bytes) = payload.recv().await { match bytes { Ok(bytes) => { - file.write_all(bytes.as_ref()).await + file.write_all(bytes.as_ref()) + .await .map_err(|e| UpdateError::Error(Box::new(e)))?; } Err(e) => { @@ -128,7 +133,8 @@ where } } - file.flush().await + file.flush() + .await .map_err(|e| UpdateError::Error(Box::new(e)))?; let file = file.into_std().await; @@ -144,50 +150,33 @@ where .map_err(|e| UpdateError::Error(Box::new(e)))? } - async fn handle_list_updates( - &self, - uuid: Uuid, - ) -> Result> { - let store = self.store.get(&uuid).await?; + async fn handle_list_updates(&self, uuid: Uuid) -> Result> { + let update_store = self.store.get(&uuid).await?; tokio::task::spawn_blocking(move || { - let result = match store { - Some(update_store) => { - let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { - Ok(processing - .map(UpdateStatus::from) - .into_iter() - .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) - .chain(aborted.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .chain(processed.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .chain(failed.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .sorted_by(|a, b| a.id().cmp(&b.id())) - .collect()) - }) - .map_err(|e| UpdateError::Error(Box::new(e)))?; - Ok(updates) - } - None => Err(UpdateError::UnexistingIndex(uuid)), - }; - result - }).await + let result = update_store + .ok_or(UpdateError::UnexistingIndex(uuid))? + .list() + .map_err(|e| UpdateError::Error(e.into()))?; + Ok(result) + }) + .await .map_err(|e| UpdateError::Error(Box::new(e)))? } - async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result> { - let store = self.store + let store = self + .store .get(&uuid) .await? .ok_or(UpdateError::UnexistingIndex(uuid))?; - let result = store.meta(id) + let result = store + .meta(id) .map_err(|e| UpdateError::Error(Box::new(e)))?; Ok(result) } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let store = self.store - .delete(&uuid) - .await?; + let store = self.store.delete(&uuid).await?; if let Some(store) = store { tokio::task::spawn(async move { diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index fe04b205d..a7a65e494 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -1,6 +1,6 @@ use std::fs::remove_file; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; use heed::{Database, Env, EnvOpenOptions}; @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use std::fs::File; use tokio::sync::mpsc; use uuid::Uuid; +use parking_lot::RwLock; use crate::index_controller::updates::*; @@ -206,7 +207,7 @@ where // to the update handler. Processing store is non persistent to be able recover // from a failure let processing = pending.processing(); - self.processing.write().unwrap().replace(processing.clone()); + self.processing.write().replace(processing.clone()); let file = File::open(&content_path)?; // Process the pending update using the provided user function. let result = handler.handle_update(processing, file)?; @@ -216,7 +217,7 @@ where // we must remove the content from the pending and processing stores and // write the *new* meta to the processed-meta store and commit. let mut wtxn = self.env.write_txn()?; - self.processing.write().unwrap().take(); + self.processing.write().take(); self.pending_meta.delete(&mut wtxn, &first_id)?; remove_file(&content_path)?; self.pending.delete(&mut wtxn, &first_id)?; @@ -232,36 +233,52 @@ where } } - /// Execute the user defined function with the meta-store iterators, the first - /// iterator is the *processed* meta one, the second the *aborted* meta one - /// and, the last is the *pending* meta one. - pub fn iter_metas(&self, mut f: F) -> heed::Result - where - F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - ) -> heed::Result, - { + pub fn list(&self) -> anyhow::Result>> { let rtxn = self.env.read_txn()?; + let mut updates = Vec::new(); - // We get the pending, processed and aborted meta iterators. - let processed_iter = self.processed_meta.iter(&rtxn)?; - let aborted_iter = self.aborted_meta.iter(&rtxn)?; - let pending_iter = self.pending_meta.iter(&rtxn)?; - let processing = self.processing.read().unwrap().clone(); - let failed_iter = self.failed_meta.iter(&rtxn)?; + let processing = self.processing.read(); + if let Some(ref processing) = *processing { + let update = UpdateStatus::from(processing.clone()); + updates.push(update); + } - // We execute the user defined function with both iterators. - (f)( - processing, - processed_iter, - aborted_iter, - pending_iter, - failed_iter, - ) + let pending = self + .pending_meta + .iter(&rtxn)? + .filter_map(Result::ok) + .filter_map(|(_, p)| (Some(p.id()) != processing.as_ref().map(|p| p.id())).then(|| p)) + .map(UpdateStatus::from); + + updates.extend(pending); + + let aborted = + self.aborted_meta.iter(&rtxn)? + .filter_map(Result::ok) + .map(|(_, p)| p) + .map(UpdateStatus::from); + + updates.extend(aborted); + + let processed = + self.processed_meta.iter(&rtxn)? + .filter_map(Result::ok) + .map(|(_, p)| p) + .map(UpdateStatus::from); + + updates.extend(processed); + + let failed = + self.failed_meta.iter(&rtxn)? + .filter_map(Result::ok) + .map(|(_, p)| p) + .map(UpdateStatus::from); + + updates.extend(failed); + + updates.sort_unstable_by(|a, b| a.id().cmp(&b.id())); + + Ok(updates) } /// Returns the update associated meta or `None` if the update doesn't exist. @@ -269,7 +286,7 @@ where let rtxn = self.env.read_txn()?; let key = BEU64::new(update_id); - if let Some(ref meta) = *self.processing.read().unwrap() { + if let Some(ref meta) = *self.processing.read() { if meta.id() == update_id { return Ok(Some(UpdateStatus::Processing(meta.clone()))); } From 40b3451a4e46cae9300c0f3e279bc459ed9a54a0 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:11:58 +0100 Subject: [PATCH 158/522] fix unexisting update store + race conditions --- meilisearch-http/src/index_controller/mod.rs | 1 + .../src/index_controller/update_actor.rs | 60 ++++++++++++++----- meilisearch-http/tests/updates/mod.rs | 1 + 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 2c181817b..318b7270c 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -144,6 +144,7 @@ impl IndexController { let name = name.unwrap(); let uuid = self.uuid_resolver.create(name.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; + let _ = self.update_handle.create(uuid).await?; let meta = IndexMetadata { name, meta }; Ok(meta) diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 8614e297b..2c16eb60b 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -47,6 +47,10 @@ enum UpdateMsg { uuid: Uuid, ret: oneshot::Sender>, }, + Create { + uuid: Uuid, + ret: oneshot::Sender>, + } } struct UpdateActor { @@ -102,7 +106,10 @@ where Some(Delete { uuid, ret }) => { let _ = ret.send(self.handle_delete(uuid).await); } - None => {} + Some(Create { uuid, ret }) => { + let _ = ret.send(self.handle_create(uuid).await); + } + None => break, } } } @@ -190,6 +197,11 @@ where Ok(()) } + + async fn handle_create(&self, uuid: Uuid) -> Result<()> { + let _ = self.store.get_or_create(uuid).await?; + Ok(()) + } } #[derive(Clone)] @@ -249,6 +261,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn create(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Create { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { @@ -282,7 +301,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle.update(meta, file)) }) - .unwrap(); + .map_err(|e| UpdateError::Error(e.into()))?; let store = e.insert(store); Ok(store.clone()) } @@ -291,22 +310,35 @@ impl UpdateStoreStore for MapUpdateStoreStore { } async fn get(&self, uuid: &Uuid) -> Result>> { - // attemps to get pre-loaded ref to the index - match self.db.read().await.get(uuid) { + let guard = self.db.read().await; + match guard.get(uuid) { Some(uuid) => Ok(Some(uuid.clone())), None => { - // otherwise we try to check if it exists, and load it. + // The index is not found in the found in the loaded indexes, so we attempt to load + // it from disk. We need to acquire a write lock **before** attempting to open the + // index, because someone could be trying to open it at the same time as us. + drop(guard); let path = self.path.clone().join(format!("updates-{}", uuid)); if path.exists() { - let index_handle = self.index_handle.clone(); - let mut options = heed::EnvOpenOptions::new(); - options.map_size(4096 * 100_000); - let store = UpdateStore::open(options, &path, move |meta, file| { - futures::executor::block_on(index_handle.update(meta, file)) - }) - .unwrap(); - self.db.write().await.insert(uuid.clone(), store.clone()); - Ok(Some(store)) + let mut guard = self.db.write().await; + match guard.entry(uuid.clone()) { + Entry::Vacant(entry) => { + // We can safely load the index + let index_handle = self.index_handle.clone(); + let mut options = heed::EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle.update(meta, file)) + }) + .map_err(|e| UpdateError::Error(e.into()))?; + let store = entry.insert(store.clone()); + Ok(Some(store.clone())) + } + Entry::Occupied(entry) => { + // The index was loaded while we attempted to to iter + Ok(Some(entry.get().clone())) + } + } } else { Ok(None) } diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 64b5b560e..713936b8c 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -46,6 +46,7 @@ async fn list_no_updates() { let index = server.index("test"); index.create(None).await; let (response, code) = index.list_updates().await; + println!("response: {}", response); assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } From 30dd790884b01d3bf43b0b38fc7e5c3439536af0 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:23:48 +0100 Subject: [PATCH 159/522] handle badly formatted index uid --- .../src/index_controller/uuid_resolver.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index d8d39d922..4fe079518 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -72,10 +72,16 @@ impl UuidResolverActor { } async fn handle_create(&self, name: String) -> Result { + if !is_index_uid_valid(&name) { + return Err(UuidError::BadlyFormatted(name)) + } self.store.create_uuid(name, true).await } async fn handle_get_or_create(&self, name: String) -> Result { + if !is_index_uid_valid(&name) { + return Err(UuidError::BadlyFormatted(name)) + } self.store.create_uuid(name, false).await } @@ -99,6 +105,10 @@ impl UuidResolverActor { } } +fn is_index_uid_valid(uid: &str) -> bool { + uid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') +} + #[derive(Clone)] pub struct UuidResolverHandle { sender: mpsc::Sender, @@ -171,6 +181,8 @@ pub enum UuidError { Heed(#[from] heed::Error), #[error("Uuid error: {0}")] Uuid(#[from] uuid::Error), + #[error("Badly formatted index uid: {0}")] + BadlyFormatted(String), } #[async_trait::async_trait] From 66b64c1f8078f2000a0e9605a564e977e0e13864 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:33:31 +0100 Subject: [PATCH 160/522] correct error on settings delete unexisting index --- meilisearch-http/src/data/updates.rs | 5 +++-- meilisearch-http/src/index_controller/mod.rs | 8 ++++++-- meilisearch-http/src/routes/settings/mod.rs | 8 ++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index c6e30ea02..31c886673 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -23,9 +23,10 @@ impl Data { pub async fn update_settings( &self, index: String, - settings: Settings + settings: Settings, + create: bool, ) -> anyhow::Result { - let update = self.index_controller.update_settings(index, settings).await?; + let update = self.index_controller.update_settings(index, settings, create).await?; Ok(update.into()) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 318b7270c..56693e204 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -129,8 +129,12 @@ impl IndexController { Ok(status) } - pub async fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { - let uuid = self.uuid_resolver.get_or_create(index_uid).await?; + pub async fn update_settings(&self, index_uid: String, settings: Settings, create: bool) -> anyhow::Result { + let uuid = if create { + self.uuid_resolver.get_or_create(index_uid).await? + } else { + self.uuid_resolver.resolve(index_uid).await? + }; let meta = UpdateMeta::Settings(settings); // Nothing so send, drop the sender right away, as not to block the update actor. let (_, receiver) = mpsc::channel(1); diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 0f904f572..4e7219b1c 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -26,7 +26,7 @@ macro_rules! make_setting_route { $attr: Some(None), ..Default::default() }; - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, false).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -48,7 +48,7 @@ macro_rules! make_setting_route { ..Default::default() }; - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, true).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -137,7 +137,7 @@ async fn update_all( index_uid: web::Path, body: web::Json, ) -> Result { - match data.update_settings(index_uid.into_inner(), body.into_inner()).await { + match data.update_settings(index_uid.into_inner(), body.into_inner(), true).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -170,7 +170,7 @@ async fn delete_all( index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, false).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) From 8617bcf8bdd723926394a7d9e9ea1c793e98dc54 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:39:16 +0100 Subject: [PATCH 161/522] add ranking rules --- meilisearch-http/src/index/mod.rs | 8 +++++++- meilisearch-http/src/index/updates.rs | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index c50c2873c..1afeb3478 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -44,11 +44,17 @@ impl Index { .map(|(k, v)| (k, v.to_string())) .collect(); + let criteria = self + .criteria(&txn)? + .into_iter() + .map(|c| c.to_string()) + .collect(); + Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), searchable_attributes: Some(Some(searchable_attributes)), faceted_attributes: Some(Some(faceted_attributes)), - criteria: None, + ranking_rules: Some(Some(criteria)), }) } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 129a340a7..70c2dfc2b 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -42,7 +42,7 @@ pub struct Settings { deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none", )] - pub criteria: Option>>, + pub ranking_rules: Option>>, } impl Settings { @@ -51,7 +51,7 @@ impl Settings { displayed_attributes: Some(None), searchable_attributes: Some(None), faceted_attributes: Some(None), - criteria: Some(None), + ranking_rules: Some(None), } } } @@ -164,7 +164,7 @@ impl Index { } // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { + if let Some(ref criteria) = settings.ranking_rules { match criteria { Some(criteria) => builder.set_criteria(criteria.clone()), None => builder.reset_criteria(), From 271c8ba991834560b35c25ac7bd3af0724e88436 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:47:29 +0100 Subject: [PATCH 162/522] change index name to uid --- meilisearch-http/src/data/mod.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 70 ++++++++++---------- meilisearch-http/tests/index/create_index.rs | 8 +-- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index b011c18b3..f2b963eec 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -90,7 +90,7 @@ impl Data { pub async fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { let settings = IndexSettings { - name: Some(name.as_ref().to_string()), + uid: Some(name.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), }; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 56693e204..c76cd8a70 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -26,7 +26,7 @@ pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { - name: String, + uid: String, #[serde(flatten)] meta: index_actor::IndexMeta, } @@ -47,7 +47,7 @@ pub enum UpdateMeta { #[derive(Clone, Debug)] pub struct IndexSettings { - pub name: Option, + pub uid: Option, pub primary_key: Option, } @@ -77,13 +77,13 @@ impl IndexController { pub async fn add_documents( &self, - index: String, + uid: String, method: milli::update::IndexDocumentsMethod, format: milli::update::UpdateFormat, mut payload: Payload, primary_key: Option, ) -> anyhow::Result { - let uuid = self.uuid_resolver.get_or_create(index).await?; + let uuid = self.uuid_resolver.get_or_create(uid).await?; let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; let (sender, receiver) = mpsc::channel(10); @@ -106,16 +106,16 @@ impl IndexController { Ok(status) } - pub async fn clear_documents(&self, index: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await?; + pub async fn clear_documents(&self, uid: String) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(uid).await?; let meta = UpdateMeta::ClearDocuments; let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; Ok(status) } - pub async fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await?; + pub async fn delete_documents(&self, uid: String, document_ids: Vec) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(uid).await?; let meta = UpdateMeta::DeleteDocuments; let (sender, receiver) = mpsc::channel(10); @@ -129,11 +129,11 @@ impl IndexController { Ok(status) } - pub async fn update_settings(&self, index_uid: String, settings: Settings, create: bool) -> anyhow::Result { + pub async fn update_settings(&self, uid: String, settings: Settings, create: bool) -> anyhow::Result { let uuid = if create { - self.uuid_resolver.get_or_create(index_uid).await? + self.uuid_resolver.get_or_create(uid).await? } else { - self.uuid_resolver.resolve(index_uid).await? + self.uuid_resolver.resolve(uid).await? }; let meta = UpdateMeta::Settings(settings); // Nothing so send, drop the sender right away, as not to block the update actor. @@ -144,36 +144,36 @@ impl IndexController { } pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { - let IndexSettings { name, primary_key } = index_settings; - let name = name.unwrap(); - let uuid = self.uuid_resolver.create(name.clone()).await?; + let IndexSettings { uid: name, primary_key } = index_settings; + let uid = name.unwrap(); + let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; let _ = self.update_handle.create(uuid).await?; - let meta = IndexMetadata { name, meta }; + let meta = IndexMetadata { uid, meta }; Ok(meta) } - pub async fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { + pub async fn delete_index(&self, uid: String) -> anyhow::Result<()> { let uuid = self.uuid_resolver - .delete(index_uid) + .delete(uid) .await?; self.update_handle.delete(uuid.clone()).await?; self.index_handle.delete(uuid).await?; Ok(()) } - pub async fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result> { let uuid = self.uuid_resolver - .resolve(index) + .resolve(uid) .await?; let result = self.update_handle.update_status(uuid, id).await?; Ok(result) } - pub async fn all_update_status(&self, index: String) -> anyhow::Result> { + pub async fn all_update_status(&self, uid: String) -> anyhow::Result> { let uuid = self.uuid_resolver - .resolve(index).await?; + .resolve(uid).await?; let result = self.update_handle.get_all_updates_status(uuid).await?; Ok(result) } @@ -183,9 +183,9 @@ impl IndexController { let mut ret = Vec::new(); - for (name, uuid) in uuids { + for (uid, uuid) in uuids { if let Some(meta) = self.index_handle.get_index_meta(uuid).await? { - let meta = IndexMetadata { name, meta }; + let meta = IndexMetadata { uid, meta }; ret.push(meta); } } @@ -193,9 +193,9 @@ impl IndexController { Ok(ret) } - pub async fn settings(&self, index: String) -> anyhow::Result { + pub async fn settings(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver - .resolve(index.clone()) + .resolve(uid.clone()) .await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) @@ -203,13 +203,13 @@ impl IndexController { pub async fn documents( &self, - index: String, + uid: String, offset: usize, limit: usize, attributes_to_retrieve: Option>, ) -> anyhow::Result> { let uuid = self.uuid_resolver - .resolve(index.clone()) + .resolve(uid.clone()) .await?; let documents = self.index_handle.documents(uuid, offset, limit, attributes_to_retrieve).await?; Ok(documents) @@ -217,33 +217,33 @@ impl IndexController { pub async fn document( &self, - index: String, + uid: String, doc_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result { let uuid = self.uuid_resolver - .resolve(index.clone()) + .resolve(uid.clone()) .await?; let document = self.index_handle.document(uuid, doc_id, attributes_to_retrieve).await?; Ok(document) } - fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { + fn update_index(&self, uid: String, index_settings: IndexSettings) -> anyhow::Result { todo!() } - pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(name).await?; + pub async fn search(&self, uid: String, query: SearchQuery) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(uid).await?; let result = self.index_handle.search(uuid, query).await?; Ok(result) } - pub async fn get_index(&self, name: String) -> anyhow::Result> { - let uuid = self.uuid_resolver.resolve(name.clone()).await?; + pub async fn get_index(&self, uid: String) -> anyhow::Result> { + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let result = self.index_handle .get_index_meta(uuid) .await? - .map(|meta| IndexMetadata { name, meta }); + .map(|meta| IndexMetadata { uid, meta }); Ok(result) } } diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index c26941b91..b7c85d748 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -7,14 +7,15 @@ async fn create_index_no_primary_key() { let index = server.index("test"); let (response, code) = index.create(None).await; + println!("response: {}", response); + assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert!(response.get("uuid").is_some()); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], Value::Null); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } #[actix_rt::test] @@ -25,12 +26,11 @@ async fn create_index_with_primary_key() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert!(response.get("uuid").is_some()); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], "primary"); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } // TODO: partial test since we are testing error, amd error is not yet fully implemented in From 7d9637861fc71c29ff287313657ca0bfc2b95d8a Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:54:38 +0100 Subject: [PATCH 163/522] fix add primary key on index creation --- meilisearch-http/src/index_controller/index_actor.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index b5d5792b2..10a8db9f4 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -486,6 +486,11 @@ impl IndexStore for HeedIndexStore { let index = spawn_blocking(move || -> Result { let index = open_index(&path, 4096 * 100_000)?; + if let Some(primary_key) = primary_key { + let mut txn = index.write_txn()?; + index.put_primary_key(&mut txn, &primary_key)?; + txn.commit()?; + } Ok(index) }) .await From e4d45b05002838564bd6ac0c0ea3fdaefe3e6266 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 00:37:43 +0100 Subject: [PATCH 164/522] fix various bugs --- .../src/index_controller/index_actor.rs | 1 + meilisearch-http/src/index_controller/mod.rs | 11 +++- .../src/index_controller/uuid_resolver.rs | 55 +------------------ meilisearch-http/tests/index/create_index.rs | 2 +- meilisearch-http/tests/index/delete_index.rs | 6 +- meilisearch-http/tests/index/get_index.rs | 6 +- .../tests/settings/get_settings.rs | 11 ++-- 7 files changed, 27 insertions(+), 65 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 10a8db9f4..8ae3f8946 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -343,6 +343,7 @@ impl IndexActor { async fn handle_get_meta(&self, uuid: Uuid) -> Result> { match self.store.get(uuid).await? { Some(index) => { + println!("geting meta yoyo"); let meta = spawn_blocking(move || IndexMeta::new(&index)) .await .map_err(|e| IndexError::Error(e.into()))??; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index c76cd8a70..0524b6e1b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -131,7 +131,16 @@ impl IndexController { pub async fn update_settings(&self, uid: String, settings: Settings, create: bool) -> anyhow::Result { let uuid = if create { - self.uuid_resolver.get_or_create(uid).await? + let uuid = self.uuid_resolver.get_or_create(uid).await?; + // We need to create the index upfront, since it would otherwise only be created when + // the update is processed. This would make calls to GET index to fail until the update + // is complete. Since this is get or create, we ignore the error when the index already + // exists. + match self.index_handle.create_index(uuid.clone(), None).await { + Ok(_) | Err(index_actor::IndexError::IndexAlreadyExists) => (), + Err(e) => return Err(e.into()), + } + uuid } else { self.uuid_resolver.resolve(uid).await? }; diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 4fe079518..aa83db2c7 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -270,7 +270,7 @@ impl UuidStore for HeedUuidStore { let uuid = Uuid::from_slice(uuid)?; db.delete(&mut txn, &name)?; txn.commit()?; - Ok(None) + Ok(Some(uuid)) } None => Ok(None) } @@ -289,57 +289,6 @@ impl UuidStore for HeedUuidStore { entries.push((name.to_owned(), uuid)) } Ok(entries) - }).await? } -} - -#[cfg(test)] -mod test { - use std::collections::HashMap; - use std::collections::hash_map::Entry; - use std::sync::Arc; - - use tokio::sync::RwLock; - - use super::*; - - struct MapUuidStore(Arc>>); - - #[async_trait::async_trait] - impl UuidStore for MapUuidStore { - async fn create_uuid(&self, name: String, err: bool) -> Result { - match self.0.write().await.entry(name) { - Entry::Occupied(entry) => { - if err { - Err(UuidError::NameAlreadyExist) - } else { - Ok(entry.get().clone()) - } - } - Entry::Vacant(entry) => { - let uuid = Uuid::new_v4(); - let uuid = entry.insert(uuid); - Ok(uuid.clone()) - } - } - } - - async fn get_uuid(&self, name: String) -> Result> { - Ok(self.0.read().await.get(&name).cloned()) - } - - async fn delete(&self, name: String) -> Result> { - Ok(self.0.write().await.remove(&name)) - } - - async fn list(&self) -> Result> { - let list = self - .0 - .read() - .await - .iter() - .map(|(name, uuid)| (name.to_owned(), uuid.clone())) - .collect(); - Ok(list) - } + }).await? } } diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index b7c85d748..219ad692b 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -70,5 +70,5 @@ async fn test_create_multiple_indexes() { assert_eq!(index1.get().await.1, 200); assert_eq!(index2.get().await.1, 200); assert_eq!(index3.get().await.1, 200); - assert_eq!(index4.get().await.1, 400); + assert_eq!(index4.get().await.1, 404); } diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 39e79daaf..b15cc306e 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -6,13 +6,17 @@ async fn create_and_delete_index() { let index = server.index("test"); let (_response, code) = index.create(None).await; + println!("response: {}", _response); + assert_eq!(code, 200); let (_response, code) = index.delete().await; + println!("response: {}", _response); + assert_eq!(code, 200); - assert_eq!(index.get().await.1, 400); + assert_eq!(index.get().await.1, 404); } #[actix_rt::test] diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index 3b8390551..a1a95eef8 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -13,12 +13,11 @@ async fn create_and_get_index() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert!(response.get("uuid").is_some()); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], Value::Null); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } // TODO: partial test since we are testing error, amd error is not yet fully implemented in @@ -30,7 +29,7 @@ async fn get_unexisting_index() { let (_response, code) = index.get().await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -55,5 +54,4 @@ async fn list_multiple_indexes() { assert_eq!(arr.len(), 2); assert!(arr.iter().find(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null).is_some()); assert!(arr.iter().find(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key").is_some()); - } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index ba5d9651c..b8217c5f7 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -43,7 +43,8 @@ async fn test_partial_update() { assert_eq!(response["displayedAttributes"],json!(["foo"])); assert_eq!(response["searchableAttributes"],json!(["*"])); - index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + let (response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + println!("resp: {}", response); index.wait_update_id(1).await; let (response, code) = index.settings().await; @@ -125,10 +126,10 @@ macro_rules! test_setting_routes { .chars() .map(|c| if c == '_' { '-' } else { c }) .collect::()); - let (_response, code) = server.service.post(url, serde_json::Value::Null).await; - assert_eq!(code, 200); - let (_response, code) = server.index("test").get().await; - assert_eq!(code, 200); + let (response, code) = server.service.post(url, serde_json::Value::Null).await; + assert_eq!(code, 200, "{}", response); + let (response, code) = server.index("test").get().await; + assert_eq!(code, 200, "{}", response); } #[actix_rt::test] From 77d5dd452fb0e0b95269fb62fa63e60470537fe3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 14:16:54 +0100 Subject: [PATCH 165/522] remove open_or_create --- meilisearch-http/src/index_controller/uuid_resolver.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index aa83db2c7..fcf417248 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -200,13 +200,6 @@ struct HeedUuidStore { db: Database, } -fn open_or_create_database(env: &Env, name: Option<&str>) -> heed::Result> { - match env.open_database(name)? { - Some(db) => Ok(db), - None => env.create_database(name), - } -} - impl HeedUuidStore { fn new(path: impl AsRef) -> anyhow::Result { let path = path.as_ref().join("index_uuids"); @@ -214,7 +207,7 @@ impl HeedUuidStore { let mut options = EnvOpenOptions::new(); options.map_size(1_073_741_824); // 1GB let env = options.open(path)?; - let db = open_or_create_database(&env, None)?; + let db = env.create_database(None)?; Ok(Self { env, db }) } } From c4846dafca3137f561357c55abbf973b87c51888 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 14:48:43 +0100 Subject: [PATCH 166/522] implement update index --- meilisearch-http/src/data/updates.rs | 15 +++--- .../src/index_controller/index_actor.rs | 54 ++++++++++++++++++- meilisearch-http/src/index_controller/mod.rs | 14 ++++- meilisearch-http/src/routes/index.rs | 2 +- meilisearch-http/tests/index/create_index.rs | 2 +- meilisearch-http/tests/index/update_index.rs | 3 +- 6 files changed, 75 insertions(+), 15 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index 31c886673..0e2c56e42 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,7 +1,7 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use actix_web::web::Payload; -use crate::index_controller::{UpdateStatus, IndexMetadata}; +use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus}; use crate::index::Settings; use super::Data; @@ -63,18 +63,17 @@ impl Data { self.index_controller.all_update_status(index.as_ref().to_string()).await } - pub fn update_index( + pub async fn update_index( &self, name: impl AsRef, primary_key: Option>, new_name: Option> ) -> anyhow::Result { - todo!() - //let settings = IndexSettings { - //name: new_name.map(|s| s.as_ref().to_string()), - //primary_key: primary_key.map(|s| s.as_ref().to_string()), - //}; + let settings = IndexSettings { + uid: new_name.map(|s| s.as_ref().to_string()), + primary_key: primary_key.map(|s| s.as_ref().to_string()), + }; - //self.index_controller.update_index(name, settings) + self.index_controller.update_index(name.as_ref().to_string(), settings).await } } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 8ae3f8946..63be1eec5 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -17,7 +17,7 @@ use tokio::task::spawn_blocking; use tokio::fs::remove_dir_all; use uuid::Uuid; -use super::get_arc_ownership_blocking; +use super::{IndexSettings, get_arc_ownership_blocking}; use super::update_handler::UpdateHandler; use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; @@ -42,6 +42,10 @@ pub struct IndexMeta { impl IndexMeta { fn new(index: &Index) -> Result { let txn = index.read_txn()?; + Self::new_txn(index, &txn) + } + + fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { let created_at = index.created_at(&txn)?; let updated_at = index.updated_at(&txn)?; let primary_key = index.primary_key(&txn)?.map(String::from); @@ -90,6 +94,11 @@ enum IndexMsg { uuid: Uuid, ret: oneshot::Sender>>, }, + UpdateIndex { + uuid: Uuid, + index_settings: IndexSettings, + ret: oneshot::Sender>, + } } struct IndexActor { @@ -109,6 +118,8 @@ pub enum IndexError { UnexistingIndex, #[error("Heed error: {0}")] HeedError(#[from] heed::Error), + #[error("Existing primary key")] + ExistingPrimaryKey, } #[async_trait::async_trait] @@ -230,6 +241,9 @@ impl IndexActor { GetMeta { uuid, ret } => { let _ = ret.send(self.handle_get_meta(uuid).await); } + UpdateIndex { uuid, index_settings, ret } => { + let _ = ret.send(self.handle_update_index(uuid, index_settings).await); + } } } @@ -352,6 +366,33 @@ impl IndexActor { None => Ok(None), } } + + async fn handle_update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { + let index = self.store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + + spawn_blocking(move || { + match index_settings.primary_key { + Some(ref primary_key) => { + let mut txn = index.write_txn()?; + if index.primary_key(&txn)?.is_some() { + return Err(IndexError::ExistingPrimaryKey) + } + index.put_primary_key(&mut txn, primary_key)?; + let meta = IndexMeta::new_txn(&index, &txn)?; + txn.commit()?; + Ok(meta) + }, + None => { + let meta = IndexMeta::new(&index)?; + Ok(meta) + }, + } + }).await + .map_err(|e| IndexError::Error(e.into()))? + } } #[derive(Clone)] @@ -459,6 +500,17 @@ impl IndexActorHandle { let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::UpdateIndex { uuid, index_settings, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct HeedIndexStore { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 0524b6e1b..1433562ff 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -9,6 +9,7 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; +use anyhow::bail; use actix_web::web::{Bytes, Payload}; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; @@ -237,8 +238,17 @@ impl IndexController { Ok(document) } - fn update_index(&self, uid: String, index_settings: IndexSettings) -> anyhow::Result { - todo!() + pub async fn update_index(&self, uid: String, index_settings: IndexSettings) -> anyhow::Result { + if index_settings.uid.is_some() { + bail!("Can't change the index uid.") + } + + let uuid = self.uuid_resolver + .resolve(uid.clone()) + .await?; + let meta = self.index_handle.update_index(uuid, index_settings).await?; + let meta = IndexMetadata { uid, meta }; + Ok(meta) } pub async fn search(&self, uid: String, query: SearchQuery) -> anyhow::Result { diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 5c6e3f5a9..68c94798a 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -95,7 +95,7 @@ async fn update_index( path: web::Path, body: web::Json, ) -> Result { - match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.name.as_ref()) { + match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.name.as_ref()).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 219ad692b..718e35899 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -28,7 +28,7 @@ async fn create_index_with_primary_key() { assert_eq!(response["uid"], "test"); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); - assert_eq!(response["createdAt"], response["updatedAt"]); + //assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], "primary"); assert_eq!(response.as_object().unwrap().len(), 4); } diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index 0078ad892..36670682a 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -13,7 +13,6 @@ async fn update_primary_key() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert!(response.get("uuid").is_some()); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); @@ -22,7 +21,7 @@ async fn update_primary_key() { assert!(created_at < updated_at); assert_eq!(response["primaryKey"], "primary"); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } #[actix_rt::test] From 2ee2e6a9b285be56c3b611de8f1020f49615c3a1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 14:57:24 +0100 Subject: [PATCH 167/522] clean project --- .../actor_index_controller/mod.rs | 127 ---- .../src/index_controller/index_actor.rs | 1 - .../local_index_controller/index_store.rs | 606 ------------------ .../local_index_controller/mod.rs | 228 ------- .../tests/documents/add_documents.rs | 1 - meilisearch-http/tests/index/create_index.rs | 1 - meilisearch-http/tests/index/delete_index.rs | 2 - .../tests/settings/get_settings.rs | 3 - meilisearch-http/tests/updates/mod.rs | 1 - 9 files changed, 970 deletions(-) delete mode 100644 meilisearch-http/src/index_controller/actor_index_controller/mod.rs delete mode 100644 meilisearch-http/src/index_controller/local_index_controller/index_store.rs delete mode 100644 meilisearch-http/src/index_controller/local_index_controller/mod.rs diff --git a/meilisearch-http/src/index_controller/actor_index_controller/mod.rs b/meilisearch-http/src/index_controller/actor_index_controller/mod.rs deleted file mode 100644 index 188d85580..000000000 --- a/meilisearch-http/src/index_controller/actor_index_controller/mod.rs +++ /dev/null @@ -1,127 +0,0 @@ -mod index_actor; -mod update_actor; -mod uuid_resolver; -mod update_store; -mod update_handler; - -use std::path::Path; - -use tokio::sync::{mpsc, oneshot}; -use uuid::Uuid; -use super::IndexMetadata; -use futures::stream::StreamExt; -use actix_web::web::Payload; -use super::UpdateMeta; -use crate::index::{SearchResult, SearchQuery}; -use actix_web::web::Bytes; - -use crate::index::Settings; -use super::UpdateStatus; - -pub struct IndexController { - uuid_resolver: uuid_resolver::UuidResolverHandle, - index_handle: index_actor::IndexActorHandle, - update_handle: update_actor::UpdateActorHandle, -} - -enum IndexControllerMsg { - CreateIndex { - uuid: Uuid, - primary_key: Option, - ret: oneshot::Sender>, - }, - Shutdown, -} - -impl IndexController { - pub fn new(path: impl AsRef) -> Self { - let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); - let index_actor = index_actor::IndexActorHandle::new(&path); - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); - Self { uuid_resolver, index_handle: index_actor, update_handle } - } - - pub async fn add_documents( - &self, - index: String, - method: milli::update::IndexDocumentsMethod, - format: milli::update::UpdateFormat, - mut payload: Payload, - primary_key: Option, - ) -> anyhow::Result { - let uuid = self.uuid_resolver.get_or_create(index).await?; - let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; - let (sender, receiver) = mpsc::channel(10); - - // It is necessary to spawn a local task to senf the payload to the update handle to - // prevent dead_locking between the update_handle::update that waits for the update to be - // registered and the update_actor that waits for the the payload to be sent to it. - tokio::task::spawn_local(async move { - while let Some(bytes) = payload.next().await { - match bytes { - Ok(bytes) => { sender.send(Ok(bytes)).await; }, - Err(e) => { - let error: Box = Box::new(e); - sender.send(Err(error)).await; }, - } - } - }); - - // This must be done *AFTER* spawning the task. - let status = self.update_handle.update(meta, receiver, uuid).await?; - Ok(status) - } - - fn clear_documents(&self, index: String) -> anyhow::Result { - todo!() - } - - fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { - todo!() - } - - fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { - todo!() - } - - pub async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { - let super::IndexSettings { name, primary_key } = index_settings; - let uuid = self.uuid_resolver.create(name.unwrap()).await?; - let index_meta = self.index_handle.create_index(uuid, primary_key).await?; - Ok(index_meta) - } - - fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { - todo!() - } - - fn swap_indices(&self, index1_uid: String, index2_uid: String) -> anyhow::Result<()> { - todo!() - } - - pub fn index(&self, name: String) -> anyhow::Result>> { - todo!() - } - - fn update_status(&self, index: String, id: u64) -> anyhow::Result> { - todo!() - } - - fn all_update_status(&self, index: String) -> anyhow::Result> { - todo!() - } - - pub fn list_indexes(&self) -> anyhow::Result> { - todo!() - } - - fn update_index(&self, name: String, index_settings: super::IndexSettings) -> anyhow::Result { - todo!() - } - - pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); - let result = self.index_handle.search(uuid, query).await?; - Ok(result) - } -} diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 63be1eec5..52a69b964 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -357,7 +357,6 @@ impl IndexActor { async fn handle_get_meta(&self, uuid: Uuid) -> Result> { match self.store.get(uuid).await? { Some(index) => { - println!("geting meta yoyo"); let meta = spawn_blocking(move || IndexMeta::new(&index)) .await .map_err(|e| IndexError::Error(e.into()))??; diff --git a/meilisearch-http/src/index_controller/local_index_controller/index_store.rs b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs deleted file mode 100644 index a690abaf4..000000000 --- a/meilisearch-http/src/index_controller/local_index_controller/index_store.rs +++ /dev/null @@ -1,606 +0,0 @@ -use std::fs::{create_dir_all, remove_dir_all}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::Duration; - -use anyhow::{bail, Context}; -use chrono::{DateTime, Utc}; -use dashmap::{mapref::entry::Entry, DashMap}; -use heed::{ - types::{ByteSlice, SerdeJson, Str}, - Database, Env, EnvOpenOptions, RoTxn, RwTxn, -}; -use log::{error, info}; -use milli::Index; -use rayon::ThreadPool; use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use super::update_handler::UpdateHandler; -use super::{UpdateMeta, UpdateResult}; -use crate::option::IndexerOpts; - -type UpdateStore = super::update_store::UpdateStore; - -#[derive(Serialize, Deserialize, Debug, PartialEq)] -pub struct IndexMeta { - update_store_size: u64, - index_store_size: u64, - pub uuid: Uuid, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -impl IndexMeta { - fn open( - &self, - path: impl AsRef, - thread_pool: Arc, - indexer_options: &IndexerOpts, - ) -> anyhow::Result<(Arc, Arc)> { - let update_path = make_update_db_path(&path, &self.uuid); - let index_path = make_index_db_path(&path, &self.uuid); - - create_dir_all(&update_path)?; - create_dir_all(&index_path)?; - - let mut options = EnvOpenOptions::new(); - options.map_size(self.index_store_size as usize); - let index = Arc::new(Index::new(options, index_path)?); - - let mut options = EnvOpenOptions::new(); - options.map_size(self.update_store_size as usize); - let handler = UpdateHandler::new(indexer_options, index.clone(), thread_pool)?; - let update_store = UpdateStore::open(options, update_path, handler)?; - - Ok((index, update_store)) - } -} - -pub struct IndexStore { - env: Env, - name_to_uuid: Database, - uuid_to_index: DashMap, Arc)>, - uuid_to_index_meta: Database>, - - thread_pool: Arc, - indexer_options: IndexerOpts, -} - -impl IndexStore { - pub fn new(path: impl AsRef, indexer_options: IndexerOpts) -> anyhow::Result { - let env = EnvOpenOptions::new() - .map_size(4096 * 100) - .max_dbs(2) - .open(path)?; - - let uuid_to_index = DashMap::new(); - let name_to_uuid = open_or_create_database(&env, Some("name_to_uid"))?; - let uuid_to_index_meta = open_or_create_database(&env, Some("uid_to_index_db"))?; - - let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(indexer_options.indexing_jobs.unwrap_or(0)) - .build()?; - let thread_pool = Arc::new(thread_pool); - - Ok(Self { - env, - name_to_uuid, - uuid_to_index, - uuid_to_index_meta, - - thread_pool, - indexer_options, - }) - } - - pub fn delete(&self, index_uid: impl AsRef) -> anyhow::Result<()> { - // we remove the references to the index from the index map so it is not accessible anymore - let mut txn = self.env.write_txn()?; - let uuid = self - .index_uuid(&txn, &index_uid)? - .with_context(|| format!("Index {:?} doesn't exist", index_uid.as_ref()))?; - self.name_to_uuid.delete(&mut txn, index_uid.as_ref())?; - self.uuid_to_index_meta.delete(&mut txn, uuid.as_bytes())?; - txn.commit()?; - // If the index was loaded (i.e it is present in the uuid_to_index map), then we need to - // close it. The process goes as follow: - // - // 1) We want to remove any pending updates from the store. - // 2) We try to get ownership on the update store so we can close it. It may take a - // couple of tries, but since the update store event loop only has a weak reference to - // itself, and we are the only other function holding a reference to it otherwise, we will - // get it eventually. - // 3) We request a closing of the update store. - // 4) We can take ownership on the index, and close it. - // 5) We remove all the files from the file system. - let index_uid = index_uid.as_ref().to_string(); - let path = self.env.path().to_owned(); - if let Some((_, (index, updates))) = self.uuid_to_index.remove(&uuid) { - std::thread::spawn(move || { - info!("Preparing for {:?} deletion.", index_uid); - // this error is non fatal, but may delay the deletion. - if let Err(e) = updates.abort_pendings() { - error!( - "error aborting pending updates when deleting index {:?}: {}", - index_uid, e - ); - } - let updates = get_arc_ownership_blocking(updates); - let close_event = updates.prepare_for_closing(); - close_event.wait(); - info!("closed update store for {:?}", index_uid); - - let index = get_arc_ownership_blocking(index); - let close_event = index.prepare_for_closing(); - close_event.wait(); - - let update_path = make_update_db_path(&path, &uuid); - let index_path = make_index_db_path(&path, &uuid); - - if let Err(e) = remove_dir_all(index_path) { - error!("error removing index {:?}: {}", index_uid, e); - } - - if let Err(e) = remove_dir_all(update_path) { - error!("error removing index {:?}: {}", index_uid, e); - } - - info!("index {:?} deleted.", index_uid); - }); - } - - Ok(()) - } - - fn index_uuid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uuid.get(txn, name.as_ref())? { - Some(bytes) => { - let uuid = Uuid::from_slice(bytes)?; - Ok(Some(uuid)) - } - None => Ok(None), - } - } - - fn retrieve_index( - &self, - txn: &RoTxn, - uid: Uuid, - ) -> anyhow::Result, Arc)>> { - match self.uuid_to_index.entry(uid.clone()) { - Entry::Vacant(entry) => match self.uuid_to_index_meta.get(txn, uid.as_bytes())? { - Some(meta) => { - let path = self.env.path(); - let (index, updates) = - meta.open(path, self.thread_pool.clone(), &self.indexer_options)?; - entry.insert((index.clone(), updates.clone())); - Ok(Some((index, updates))) - } - None => Ok(None), - }, - Entry::Occupied(entry) => { - let (index, updates) = entry.get(); - Ok(Some((index.clone(), updates.clone()))) - } - } - } - - fn get_index_txn( - &self, - txn: &RoTxn, - name: impl AsRef, - ) -> anyhow::Result, Arc)>> { - match self.index_uuid(&txn, name)? { - Some(uid) => self.retrieve_index(&txn, uid), - None => Ok(None), - } - } - - pub fn index( - &self, - name: impl AsRef, - ) -> anyhow::Result, Arc)>> { - let txn = self.env.read_txn()?; - self.get_index_txn(&txn, name) - } - - /// Use this function to perform an update on an index. - /// This function also puts a lock on what index is allowed to perform an update. - pub fn update_index(&self, name: impl AsRef, f: F) -> anyhow::Result<(T, IndexMeta)> - where - F: FnOnce(&Index) -> anyhow::Result, - { - let mut txn = self.env.write_txn()?; - let (index, _) = self - .get_index_txn(&txn, &name)? - .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; - let result = f(index.as_ref()); - match result { - Ok(ret) => { - let meta = self.update_meta(&mut txn, name, |meta| meta.updated_at = Utc::now())?; - txn.commit()?; - Ok((ret, meta)) - } - Err(e) => Err(e), - } - } - - pub fn index_with_meta( - &self, - name: impl AsRef, - ) -> anyhow::Result, IndexMeta)>> { - let txn = self.env.read_txn()?; - let uuid = self.index_uuid(&txn, &name)?; - match uuid { - Some(uuid) => { - let meta = self - .uuid_to_index_meta - .get(&txn, uuid.as_bytes())? - .with_context(|| { - format!("unable to retrieve metadata for index {:?}", name.as_ref()) - })?; - let (index, _) = self - .retrieve_index(&txn, uuid)? - .with_context(|| format!("unable to retrieve index {:?}", name.as_ref()))?; - Ok(Some((index, meta))) - } - None => Ok(None), - } - } - - fn update_meta( - &self, - txn: &mut RwTxn, - name: impl AsRef, - f: F, - ) -> anyhow::Result - where - F: FnOnce(&mut IndexMeta), - { - let uuid = self - .index_uuid(txn, &name)? - .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; - let mut meta = self - .uuid_to_index_meta - .get(txn, uuid.as_bytes())? - .with_context(|| format!("couldn't retrieve metadata for index {:?}", name.as_ref()))?; - f(&mut meta); - self.uuid_to_index_meta.put(txn, uuid.as_bytes(), &meta)?; - Ok(meta) - } - - pub fn get_or_create_index( - &self, - name: impl AsRef, - update_size: u64, - index_size: u64, - ) -> anyhow::Result<(Arc, Arc)> { - let mut txn = self.env.write_txn()?; - match self.get_index_txn(&txn, name.as_ref())? { - Some(res) => Ok(res), - None => { - let uuid = Uuid::new_v4(); - let (index, updates, _) = - self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; - // If we fail to commit the transaction, we must delete the database from the - // file-system. - if let Err(e) = txn.commit() { - self.clean_db(uuid); - return Err(e)?; - } - Ok((index, updates)) - } - } - } - - // Remove all the files and data associated with a db uuid. - fn clean_db(&self, uuid: Uuid) { - let update_db_path = make_update_db_path(self.env.path(), &uuid); - let index_db_path = make_index_db_path(self.env.path(), &uuid); - - remove_dir_all(update_db_path).expect("Failed to clean database"); - remove_dir_all(index_db_path).expect("Failed to clean database"); - - self.uuid_to_index.remove(&uuid); - } - - fn create_index_txn( - &self, - txn: &mut RwTxn, - uuid: Uuid, - name: impl AsRef, - update_store_size: u64, - index_store_size: u64, - ) -> anyhow::Result<(Arc, Arc, IndexMeta)> { - let created_at = Utc::now(); - let updated_at = created_at; - let meta = IndexMeta { - update_store_size, - index_store_size, - uuid: uuid.clone(), - created_at, - updated_at, - }; - - self.name_to_uuid.put(txn, name.as_ref(), uuid.as_bytes())?; - self.uuid_to_index_meta.put(txn, uuid.as_bytes(), &meta)?; - - let path = self.env.path(); - let (index, update_store) = - match meta.open(path, self.thread_pool.clone(), &self.indexer_options) { - Ok(res) => res, - Err(e) => { - self.clean_db(uuid); - return Err(e); - } - }; - - self.uuid_to_index - .insert(uuid, (index.clone(), update_store.clone())); - - Ok((index, update_store, meta)) - } - - /// Same as `get_or_create`, but returns an error if the index already exists. - pub fn create_index( - &self, - name: impl AsRef, - update_size: u64, - index_size: u64, - ) -> anyhow::Result<(Arc, Arc, IndexMeta)> { - let uuid = Uuid::new_v4(); - let mut txn = self.env.write_txn()?; - - if self.name_to_uuid.get(&txn, name.as_ref())?.is_some() { - bail!("index {:?} already exists", name.as_ref()) - } - - let result = self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; - // If we fail to commit the transaction, we must delete the database from the - // file-system. - if let Err(e) = txn.commit() { - self.clean_db(uuid); - return Err(e)?; - } - Ok(result) - } - - /// Returns each index associated with its metadata: - /// (index_name, IndexMeta, primary_key) - /// This method will force all the indexes to be loaded. - pub fn list_indexes(&self) -> anyhow::Result)>> { - let txn = self.env.read_txn()?; - let metas = self.name_to_uuid.iter(&txn)?.filter_map(|entry| { - entry - .map_err(|e| { - error!("error decoding entry while listing indexes: {}", e); - e - }) - .ok() - }); - let mut indexes = Vec::new(); - for (name, uuid) in metas { - // get index to retrieve primary key - let (index, _) = self - .get_index_txn(&txn, name)? - .with_context(|| format!("could not load index {:?}", name))?; - let primary_key = index.primary_key(&index.read_txn()?)?.map(String::from); - // retieve meta - let meta = self - .uuid_to_index_meta - .get(&txn, &uuid)? - .with_context(|| format!("could not retieve meta for index {:?}", name))?; - indexes.push((name.to_owned(), meta, primary_key)); - } - Ok(indexes) - } -} - -// Loops on an arc to get ownership on the wrapped value. This method sleeps 100ms before retrying. -fn get_arc_ownership_blocking(mut item: Arc) -> T { - loop { - match Arc::try_unwrap(item) { - Ok(item) => return item, - Err(item_arc) => { - item = item_arc; - std::thread::sleep(Duration::from_millis(100)); - continue; - } - } - } -} - -fn open_or_create_database( - env: &Env, - name: Option<&str>, -) -> anyhow::Result> { - match env.open_database::(name)? { - Some(db) => Ok(db), - None => Ok(env.create_database::(name)?), - } -} - -fn make_update_db_path(path: impl AsRef, uuid: &Uuid) -> PathBuf { - let mut path = path.as_ref().to_path_buf(); - path.push(format!("update{}", uuid)); - path -} - -fn make_index_db_path(path: impl AsRef, uuid: &Uuid) -> PathBuf { - let mut path = path.as_ref().to_path_buf(); - path.push(format!("index{}", uuid)); - path -} - -#[cfg(test)] -mod test { - use super::*; - use std::path::PathBuf; - - #[test] - fn test_make_update_db_path() { - let uuid = Uuid::new_v4(); - assert_eq!( - make_update_db_path("/home", &uuid), - PathBuf::from(format!("/home/update{}", uuid)) - ); - } - - #[test] - fn test_make_index_db_path() { - let uuid = Uuid::new_v4(); - assert_eq!( - make_index_db_path("/home", &uuid), - PathBuf::from(format!("/home/index{}", uuid)) - ); - } - - mod index_store { - use super::*; - - #[test] - fn test_index_uuid() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - - let name = "foobar"; - let txn = store.env.read_txn().unwrap(); - // name is not found if the uuid in not present in the db - assert!(store.index_uuid(&txn, &name).unwrap().is_none()); - drop(txn); - - // insert an uuid in the the name_to_uuid_db: - let uuid = Uuid::new_v4(); - let mut txn = store.env.write_txn().unwrap(); - store - .name_to_uuid - .put(&mut txn, &name, uuid.as_bytes()) - .unwrap(); - txn.commit().unwrap(); - - // check that the uuid is there - let txn = store.env.read_txn().unwrap(); - assert_eq!(store.index_uuid(&txn, &name).unwrap(), Some(uuid)); - } - - #[test] - fn test_retrieve_index() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - let uuid = Uuid::new_v4(); - - let txn = store.env.read_txn().unwrap(); - assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); - - let created_at = Utc::now(); - let updated_at = created_at; - - let meta = IndexMeta { - update_store_size: 4096 * 100, - index_store_size: 4096 * 100, - uuid: uuid.clone(), - created_at, - updated_at, - }; - let mut txn = store.env.write_txn().unwrap(); - store - .uuid_to_index_meta - .put(&mut txn, uuid.as_bytes(), &meta) - .unwrap(); - txn.commit().unwrap(); - - // the index cache should be empty - assert!(store.uuid_to_index.is_empty()); - - let txn = store.env.read_txn().unwrap(); - assert!(store.retrieve_index(&txn, uuid).unwrap().is_some()); - assert_eq!(store.uuid_to_index.len(), 1); - } - - #[test] - fn test_index() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - let name = "foobar"; - - assert!(store.index(&name).unwrap().is_none()); - - let created_at = Utc::now(); - let updated_at = created_at; - - let uuid = Uuid::new_v4(); - let meta = IndexMeta { - update_store_size: 4096 * 100, - index_store_size: 4096 * 100, - uuid: uuid.clone(), - created_at, - updated_at, - }; - let mut txn = store.env.write_txn().unwrap(); - store - .name_to_uuid - .put(&mut txn, &name, uuid.as_bytes()) - .unwrap(); - store - .uuid_to_index_meta - .put(&mut txn, uuid.as_bytes(), &meta) - .unwrap(); - txn.commit().unwrap(); - - assert!(store.index(&name).unwrap().is_some()); - } - - #[test] - fn test_get_or_create_index() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - let name = "foobar"; - - let update_store_size = 4096 * 100; - let index_store_size = 4096 * 100; - store - .get_or_create_index(&name, update_store_size, index_store_size) - .unwrap(); - let txn = store.env.read_txn().unwrap(); - let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); - assert_eq!(store.uuid_to_index.len(), 1); - assert!(uuid.is_some()); - let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = store - .uuid_to_index_meta - .get(&txn, uuid.as_bytes()) - .unwrap() - .unwrap(); - assert_eq!(meta.update_store_size, update_store_size); - assert_eq!(meta.index_store_size, index_store_size); - assert_eq!(meta.uuid, uuid); - } - - #[test] - fn test_create_index() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - let name = "foobar"; - - let update_store_size = 4096 * 100; - let index_store_size = 4096 * 100; - let uuid = Uuid::new_v4(); - let mut txn = store.env.write_txn().unwrap(); - store - .create_index_txn(&mut txn, uuid, name, update_store_size, index_store_size) - .unwrap(); - let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); - assert_eq!(store.uuid_to_index.len(), 1); - assert!(uuid.is_some()); - let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = store - .uuid_to_index_meta - .get(&txn, uuid.as_bytes()) - .unwrap() - .unwrap(); - assert_eq!(meta.update_store_size, update_store_size); - assert_eq!(meta.index_store_size, index_store_size); - assert_eq!(meta.uuid, uuid); - } - } -} diff --git a/meilisearch-http/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs deleted file mode 100644 index 8ac600f5f..000000000 --- a/meilisearch-http/src/index_controller/local_index_controller/mod.rs +++ /dev/null @@ -1,228 +0,0 @@ -mod update_store; -mod index_store; -mod update_handler; - -use std::path::Path; -use std::sync::Arc; - -use anyhow::{bail, Context}; -use itertools::Itertools; -use milli::Index; - -use crate::option::IndexerOpts; -use index_store::IndexStore; -use super::IndexController; -use super::updates::UpdateStatus; -use super::{UpdateMeta, UpdateResult, IndexMetadata, IndexSettings}; - -pub struct LocalIndexController { - indexes: IndexStore, - update_db_size: u64, - index_db_size: u64, -} - -impl LocalIndexController { - pub fn new( - path: impl AsRef, - opt: IndexerOpts, - index_db_size: u64, - update_db_size: u64, - ) -> anyhow::Result { - let indexes = IndexStore::new(path, opt)?; - Ok(Self { indexes, index_db_size, update_db_size }) - } -} - -impl IndexController for LocalIndexController { - fn add_documents>( - &self, - index: S, - method: milli::update::IndexDocumentsMethod, - format: milli::update::UpdateFormat, - data: &[u8], - primary_key: Option, - ) -> anyhow::Result> { - let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; - let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; - let pending = update_store.register_update(meta, data)?; - Ok(pending.into()) - } - - fn update_settings>( - &self, - index: S, - settings: super::Settings - ) -> anyhow::Result> { - let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; - let meta = UpdateMeta::Settings(settings); - let pending = update_store.register_update(meta, &[])?; - Ok(pending.into()) - } - - fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { - let index_name = index_settings.name.context("Missing name for index")?; - let (index, _, meta) = self.indexes.create_index(&index_name, self.update_db_size, self.index_db_size)?; - if let Some(ref primary_key) = index_settings.primary_key { - if let Err(e) = update_primary_key(index, primary_key).context("error creating index") { - // TODO: creating index could not be completed, delete everything. - Err(e)? - } - } - - let meta = IndexMetadata { - uid: index_name, - uuid: meta.uuid.clone(), - created_at: meta.created_at, - updated_at: meta.created_at, - primary_key: index_settings.primary_key, - }; - - Ok(meta) - } - - fn delete_index>(&self, index_uid: S) -> anyhow::Result<()> { - self.indexes.delete(index_uid) - } - - fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { - todo!() - } - - fn index(&self, name: impl AsRef) -> anyhow::Result>> { - let index = self.indexes.index(name)?.map(|(i, _)| i); - Ok(index) - } - - fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>> { - match self.indexes.index(&index)? { - Some((_, update_store)) => Ok(update_store.meta(id)?), - None => bail!("index {:?} doesn't exist", index.as_ref()), - } - } - - fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>> { - match self.indexes.index(&index)? { - Some((_, update_store)) => { - let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { - Ok(processing - .map(UpdateStatus::from) - .into_iter() - .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) - .chain(aborted.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .chain(processed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .chain(failed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .sorted_by(|a, b| a.id().cmp(&b.id())) - .collect()) - })?; - Ok(updates) - } - None => bail!("index {} doesn't exist.", index.as_ref()), - } - - } - - fn list_indexes(&self) -> anyhow::Result> { - let metas = self.indexes.list_indexes()?; - let mut output_meta = Vec::new(); - for (uid, meta, primary_key) in metas { - let created_at = meta.created_at; - let uuid = meta.uuid; - let updated_at = self - .all_update_status(&uid)? - .iter() - .filter_map(|u| u.processed().map(|u| u.processed_at)) - .max() - .unwrap_or(created_at); - - let index_meta = IndexMetadata { - uid, - created_at, - updated_at, - uuid, - primary_key, - }; - output_meta.push(index_meta); - } - Ok(output_meta) - } - - fn update_index(&self, uid: impl AsRef, index_settings: IndexSettings) -> anyhow::Result { - if index_settings.name.is_some() { - bail!("can't update an index name.") - } - - let (primary_key, meta) = match index_settings.primary_key { - Some(ref primary_key) => { - self.indexes - .update_index(&uid, |index| { - let mut txn = index.write_txn()?; - if index.primary_key(&txn)?.is_some() { - bail!("primary key already exists.") - } - index.put_primary_key(&mut txn, primary_key)?; - txn.commit()?; - Ok(Some(primary_key.clone())) - })? - }, - None => { - let (index, meta) = self.indexes - .index_with_meta(&uid)? - .with_context(|| format!("index {:?} doesn't exist.", uid.as_ref()))?; - let primary_key = index - .primary_key(&index.read_txn()?)? - .map(String::from); - (primary_key, meta) - }, - }; - - Ok(IndexMetadata { - uid: uid.as_ref().to_string(), - uuid: meta.uuid.clone(), - created_at: meta.created_at, - updated_at: meta.updated_at, - primary_key, - }) - } - - fn clear_documents(&self, index: impl AsRef) -> anyhow::Result { - let (_, update_store) = self.indexes.index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let meta = UpdateMeta::ClearDocuments; - let pending = update_store.register_update(meta, &[])?; - Ok(pending.into()) - } - - fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result { - let (_, update_store) = self.indexes.index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let meta = UpdateMeta::DeleteDocuments; - let content = serde_json::to_vec(&document_ids)?; - let pending = update_store.register_update(meta, &content)?; - Ok(pending.into()) - } -} - -fn update_primary_key(index: impl AsRef, primary_key: impl AsRef) -> anyhow::Result<()> { - let index = index.as_ref(); - let mut txn = index.write_txn()?; - if index.primary_key(&txn)?.is_some() { - bail!("primary key already set.") - } - index.put_primary_key(&mut txn, primary_key.as_ref())?; - txn.commit()?; - Ok(()) -} - -#[cfg(test)] -mod test { - use super::*; - use tempfile::tempdir; - use crate::make_index_controller_tests; - - make_index_controller_tests!({ - let options = IndexerOpts::default(); - let path = tempdir().unwrap(); - let size = 4096 * 100; - LocalIndexController::new(path, options, size, size).unwrap() - }); -} diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 63724af18..37b06f46f 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -28,7 +28,6 @@ async fn add_documents_no_index_creation() { let (response, code) = index.get_update(0).await; assert_eq!(code, 200); - println!("response: {}", response); assert_eq!(response["status"], "processed"); assert_eq!(response["updateId"], 0); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 718e35899..0bf2f15a5 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -7,7 +7,6 @@ async fn create_index_no_primary_key() { let index = server.index("test"); let (response, code) = index.create(None).await; - println!("response: {}", response); assert_eq!(code, 200); assert_eq!(response["uid"], "test"); diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index b15cc306e..5bc78950f 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -6,13 +6,11 @@ async fn create_and_delete_index() { let index = server.index("test"); let (_response, code) = index.create(None).await; - println!("response: {}", _response); assert_eq!(code, 200); let (_response, code) = index.delete().await; - println!("response: {}", _response); assert_eq!(code, 200); diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index b8217c5f7..feefe830f 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -36,7 +36,6 @@ async fn test_partial_update() { let server = Server::new().await; let index = server.index("test"); let (response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; - println!("response: {}", response); index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); @@ -44,7 +43,6 @@ async fn test_partial_update() { assert_eq!(response["searchableAttributes"],json!(["*"])); let (response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; - println!("resp: {}", response); index.wait_update_id(1).await; let (response, code) = index.settings().await; @@ -96,7 +94,6 @@ async fn update_setting_unexisting_index_invalid_uid() { let server = Server::new().await; let index = server.index("test##! "); let (_response, code) = index.update_settings(json!({})).await; - println!("response: {}", _response); assert_eq!(code, 400); } diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 713936b8c..64b5b560e 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -46,7 +46,6 @@ async fn list_no_updates() { let index = server.index("test"); index.create(None).await; let (response, code) = index.list_updates().await; - println!("response: {}", response); assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } From 49b74b587a774de5dd21891864198e2d9d5a69af Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 17:44:39 +0100 Subject: [PATCH 168/522] enable jemalloc only on linux --- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/index_controller/mod.rs | 16 +++------------- .../src/index_controller/update_actor.rs | 2 -- meilisearch-http/src/lib.rs | 5 ----- 4 files changed, 4 insertions(+), 21 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 7b09b3042..e88556f0a 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -81,5 +81,5 @@ urlencoding = "1.1.1" [features] default = ["sentry"] -[target.'cfg(unix)'.dependencies] +[target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 1433562ff..5dc3374a0 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -14,9 +14,8 @@ use actix_web::web::{Bytes, Payload}; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::mpsc; use tokio::time::sleep; -use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; use crate::index::{SearchResult, SearchQuery, Document}; @@ -59,15 +58,6 @@ pub struct IndexController { update_handle: update_actor::UpdateActorHandle, } -enum IndexControllerMsg { - CreateIndex { - uuid: Uuid, - primary_key: Option, - ret: oneshot::Sender>, - }, - Shutdown, -} - impl IndexController { pub fn new(path: impl AsRef) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; @@ -94,10 +84,10 @@ impl IndexController { tokio::task::spawn_local(async move { while let Some(bytes) = payload.next().await { match bytes { - Ok(bytes) => { sender.send(Ok(bytes)).await; }, + Ok(bytes) => { let _ = sender.send(Ok(bytes)).await; }, Err(e) => { let error: Box = Box::new(e); - sender.send(Err(error)).await; }, + let _ = sender.send(Err(error)).await; }, } } }); diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 2c16eb60b..0eb473065 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -144,8 +144,6 @@ where .await .map_err(|e| UpdateError::Error(Box::new(e)))?; - let file = file.into_std().await; - tokio::task::spawn_blocking(move || { let result = update_store .register_update(meta, path, uuid) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index bd7379d7c..da1aee746 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,8 +1,3 @@ -#![allow(clippy::or_fun_call)] -#![allow(unused_must_use)] -#![allow(unused_variables)] -#![allow(dead_code)] - pub mod data; pub mod error; pub mod helpers; From 99c89cf2ba9bf96d53708442552af63a1c8ca520 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 13 Mar 2021 10:09:10 +0100 Subject: [PATCH 169/522] use options max db sizes --- meilisearch-http/src/data/mod.rs | 4 +++- .../src/index_controller/index_actor.rs | 17 ++++++++++------- meilisearch-http/src/index_controller/mod.rs | 6 +++--- .../src/index_controller/update_actor.rs | 14 +++++++++----- meilisearch-http/tests/settings/get_settings.rs | 4 ++-- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index f2b963eec..176422efd 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -60,7 +60,9 @@ impl Data { let path = options.db_path.clone(); create_dir_all(&path)?; - let index_controller = IndexController::new(&path)?; + let index_size = options.max_mdb_size.get_bytes() as usize; + let update_store_size = options.max_udb_size.get_bytes() as usize; + let index_controller = IndexController::new(&path, index_size, update_store_size)?; let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 52a69b964..ee7a9e782 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -401,11 +401,11 @@ pub struct IndexActorHandle { } impl IndexActorHandle { - pub fn new(path: impl AsRef) -> anyhow::Result { + pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { let (read_sender, read_receiver) = mpsc::channel(100); let (write_sender, write_receiver) = mpsc::channel(100); - let store = HeedIndexStore::new(path)?; + let store = HeedIndexStore::new(path, index_size)?; let actor = IndexActor::new(read_receiver, write_receiver, store)?; tokio::task::spawn(actor.run()); Ok(Self { @@ -416,8 +416,7 @@ impl IndexActorHandle { pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::CreateIndex { - ret, + let msg = IndexMsg::CreateIndex { ret, uuid, primary_key, }; @@ -515,15 +514,17 @@ impl IndexActorHandle { struct HeedIndexStore { index_store: AsyncMap, path: PathBuf, + index_size: usize, } impl HeedIndexStore { - fn new(path: impl AsRef) -> anyhow::Result { + fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { let path = path.as_ref().join("indexes/"); let index_store = Arc::new(RwLock::new(HashMap::new())); Ok(Self { index_store, path, + index_size, }) } } @@ -536,8 +537,9 @@ impl IndexStore for HeedIndexStore { return Err(IndexError::IndexAlreadyExists); } + let index_size = self.index_size; let index = spawn_blocking(move || -> Result { - let index = open_index(&path, 4096 * 100_000)?; + let index = open_index(&path, index_size)?; if let Some(primary_key) = primary_key { let mut txn = index.write_txn()?; index.put_primary_key(&mut txn, &primary_key)?; @@ -565,7 +567,8 @@ impl IndexStore for HeedIndexStore { return Ok(None); } - let index = spawn_blocking(|| open_index(path, 4096 * 100_000)) + let index_size = self.index_size; + let index = spawn_blocking(move || open_index(path, index_size)) .await .map_err(|e| IndexError::Error(e.into()))??; self.index_store diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 5dc3374a0..d82b304a2 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -59,10 +59,10 @@ pub struct IndexController { } impl IndexController { - pub fn new(path: impl AsRef) -> anyhow::Result { + pub fn new(path: impl AsRef, index_size: usize, update_store_size: usize) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; - let index_actor = index_actor::IndexActorHandle::new(&path)?; - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path)?; + let index_actor = index_actor::IndexActorHandle::new(&path, index_size)?; + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path, update_store_size)?; Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 0eb473065..2f66e8d20 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -211,10 +211,10 @@ impl UpdateActorHandle where D: AsRef<[u8]> + Sized + 'static + Sync + Send, { - pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> anyhow::Result { + pub fn new(index_handle: IndexActorHandle, path: impl AsRef, update_store_size: usize) -> anyhow::Result { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); - let store = MapUpdateStoreStore::new(index_handle, &path); + let store = MapUpdateStoreStore::new(index_handle, &path, update_store_size); let actor = UpdateActor::new(store, receiver, path)?; tokio::task::spawn(actor.run()); @@ -272,16 +272,18 @@ struct MapUpdateStoreStore { db: Arc>>>, index_handle: IndexActorHandle, path: PathBuf, + update_store_size: usize, } impl MapUpdateStoreStore { - fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { + fn new(index_handle: IndexActorHandle, path: impl AsRef, update_store_size: usize) -> Self { let db = Arc::new(RwLock::new(HashMap::new())); let path = path.as_ref().to_owned(); Self { db, index_handle, path, + update_store_size, } } } @@ -292,7 +294,8 @@ impl UpdateStoreStore for MapUpdateStoreStore { match self.db.write().await.entry(uuid) { Entry::Vacant(e) => { let mut options = heed::EnvOpenOptions::new(); - options.map_size(4096 * 100_000); + let update_store_size = self.update_store_size; + options.map_size(update_store_size); let path = self.path.clone().join(format!("updates-{}", e.key())); create_dir_all(&path).unwrap(); let index_handle = self.index_handle.clone(); @@ -324,7 +327,8 @@ impl UpdateStoreStore for MapUpdateStoreStore { // We can safely load the index let index_handle = self.index_handle.clone(); let mut options = heed::EnvOpenOptions::new(); - options.map_size(4096 * 100_000); + let update_store_size = self.update_store_size; + options.map_size(update_store_size); let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle.update(meta, file)) }) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index feefe830f..e00840f9b 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -35,14 +35,14 @@ async fn update_settings_unknown_field() { async fn test_partial_update() { let server = Server::new().await; let index = server.index("test"); - let (response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + let (_response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); assert_eq!(response["displayedAttributes"],json!(["foo"])); assert_eq!(response["searchableAttributes"],json!(["*"])); - let (response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + let (_response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; index.wait_update_id(1).await; let (response, code) = index.settings().await; From adc71a70ce3e93a399fab47aff8f273f49063c86 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 10:17:41 +0100 Subject: [PATCH 170/522] fix displayed attributes in document retrieval --- Cargo.lock | 78 ++++++++++++++----------------- meilisearch-http/src/index/mod.rs | 65 ++++++++++++++------------ 2 files changed, 69 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3620a6885..ba4e8580e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,7 +160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" dependencies = [ "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -170,7 +170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -365,7 +365,7 @@ checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -464,7 +464,7 @@ checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -475,7 +475,7 @@ checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -655,9 +655,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byte-unit" -version = "4.0.9" +version = "4.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8758c32833faaae35b24a73d332e62d0528e89076ae841c63940e37008b153" +checksum = "b9520900471c3a9bbcfe0fd4c7b6bcfeff41b20a76cf91c59b7474b09be1ee27" dependencies = [ "utf8-width", ] @@ -927,7 +927,7 @@ checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -984,7 +984,7 @@ dependencies = [ "heck", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -1031,7 +1031,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "synstructure", ] @@ -1172,7 +1172,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -2198,7 +2198,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -2276,7 +2276,7 @@ checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -2287,7 +2287,7 @@ checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -2329,7 +2329,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "version_check", ] @@ -2571,14 +2571,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.3" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -2592,9 +2591,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" [[package]] name = "remove_dir_all" @@ -2845,7 +2844,7 @@ checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3035,7 +3034,7 @@ dependencies = [ "quote 1.0.9", "serde", "serde_derive", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3051,7 +3050,7 @@ dependencies = [ "serde_derive", "serde_json", "sha1", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3087,7 +3086,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3103,9 +3102,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717" +checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", @@ -3129,7 +3128,7 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "unicode-xid 0.2.1", ] @@ -3203,16 +3202,7 @@ checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", -] - -[[package]] -name = "thread_local" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" -dependencies = [ - "once_cell", + "syn 1.0.64", ] [[package]] @@ -3270,7 +3260,7 @@ dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", "standback", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3348,7 +3338,7 @@ checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3486,9 +3476,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" [[package]] name = "ucd-trie" @@ -3665,7 +3655,7 @@ dependencies = [ "log", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "wasm-bindgen-shared", ] @@ -3699,7 +3689,7 @@ checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3860,7 +3850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ "proc-macro2 1.0.24", - "syn 1.0.63", + "syn 1.0.64", "synstructure", ] diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 1afeb3478..991746db3 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,15 +1,15 @@ mod search; mod updates; -use std::sync::Arc; use std::ops::Deref; +use std::sync::Arc; use anyhow::{bail, Context}; -use serde_json::{Value, Map}; use milli::obkv_to_json; +use serde_json::{Map, Value}; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; -pub use updates::{Settings, Facets, UpdateResult}; +pub use updates::{Facets, Settings, UpdateResult}; pub type Document = Map; @@ -76,7 +76,10 @@ impl Index { .iter() .filter_map(|f| fields_ids_map.id(f.as_ref())) .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), + None => match self.displayed_fields_ids(&txn)? { + Some(fields) => fields, + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }, }; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -97,36 +100,38 @@ impl Index { doc_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result> { - let txn = self.read_txn()?; + let txn = self.read_txn()?; - let fields_ids_map = self.fields_ids_map(&txn)?; + let fields_ids_map = self.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => match self.displayed_fields_ids(&txn)? { + Some(fields) => fields, None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + }, + }; + let internal_id = self + .external_documents_ids(&txn)? + .get(doc_id.as_bytes()) + .with_context(|| format!("Document with id {} not found", doc_id))?; - let internal_id = self - .external_documents_ids(&txn)? - .get(doc_id.as_bytes()) - .with_context(|| format!("Document with id {} not found", doc_id))?; + let document = self + .documents(&txn, std::iter::once(internal_id))? + .into_iter() + .next() + .map(|(_, d)| d); - let document = self - .documents(&txn, std::iter::once(internal_id))? - .into_iter() - .next() - .map(|(_, d)| d); - - match document { - Some(document) => Ok(obkv_to_json( - &attributes_to_retrieve_ids, - &fields_ids_map, - document, - )?), - None => bail!("Document with id {} not found", doc_id), - } + match document { + Some(document) => Ok(obkv_to_json( + &attributes_to_retrieve_ids, + &fields_ids_map, + document, + )?), + None => bail!("Document with id {} not found", doc_id), + } } } From 77c0a0fba5a4e3c917719c0f0b3fd24cf2aacf1d Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 10:36:12 +0100 Subject: [PATCH 171/522] add test get document displayed attributes --- meilisearch-http/src/index/mod.rs | 1 + .../tests/documents/get_documents.rs | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 991746db3..fd5aa3004 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -114,6 +114,7 @@ impl Index { None => fields_ids_map.iter().map(|(id, _)| id).collect(), }, }; + let internal_id = self .external_documents_ids(&txn)? .get(doc_id.as_bytes()) diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index 5affb8a7a..c9ed83316 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -1,6 +1,8 @@ use crate::common::Server; use crate::common::GetAllDocumentsOptions; +use serde_json::json; + // TODO: partial test since we are testing error, amd error is not yet fully implemented in // transplant #[actix_rt::test] @@ -147,3 +149,22 @@ async fn test_get_all_documents_attributes_to_retrieve() { assert_eq!(response.as_array().unwrap().len(), 20); assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 2); } + +#[actix_rt::test] +async fn get_documents_displayed_attributes() { + let server = Server::new().await; + let index = server.index("test"); + index.update_settings(json!({"displayedAttributes": ["gender"]})).await; + index.load_test_set().await; + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 1); + assert!(response.as_array().unwrap()[0].as_object().unwrap().get("gender").is_some()); + + let (response, code) = index.get_document(0, None).await; + assert_eq!(code, 200); + assert_eq!(response.as_object().unwrap().keys().count(), 1); + assert!(response.as_object().unwrap().get("gender").is_some()); +} From c079f60346adf23a11d02bde7c1f2faf606310cf Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 11:01:14 +0100 Subject: [PATCH 172/522] fixup! fix displayed attributes in document retrieval --- meilisearch-http/src/index/mod.rs | 51 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index fd5aa3004..06f1140a3 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,6 +1,7 @@ mod search; mod updates; +use std::collections::HashSet; use std::ops::Deref; use std::sync::Arc; @@ -70,17 +71,7 @@ impl Index { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => match self.displayed_fields_ids(&txn)? { - Some(fields) => fields, - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }, - }; + let fields_to_display = self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -88,7 +79,7 @@ impl Index { for entry in iter { let (_id, obkv) = entry?; - let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; + let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?; documents.push(object); } @@ -104,16 +95,7 @@ impl Index { let fields_ids_map = self.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => match self.displayed_fields_ids(&txn)? { - Some(fields) => fields, - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }, - }; + let fields_to_display = self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; let internal_id = self .external_documents_ids(&txn)? @@ -128,11 +110,34 @@ impl Index { match document { Some(document) => Ok(obkv_to_json( - &attributes_to_retrieve_ids, + &fields_to_display, &fields_ids_map, document, )?), None => bail!("Document with id {} not found", doc_id), } } + + fn fields_to_display>( + &self, + txn: &heed::RoTxn, + attributes_to_retrieve: Option>, + fields_ids_map: &milli::FieldsIdsMap, + ) -> anyhow::Result> { + let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { + Some(ids) => ids.into_iter().collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid)); + Ok(displayed_fields_ids) + } } From fcf1d4e922030b1fc64331933922aa7eaf075cb8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 12:20:33 +0100 Subject: [PATCH 173/522] fix displayed attributes in search --- meilisearch-http/src/index/search.rs | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index fb830ec60..aa2fe0715 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -78,29 +78,13 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let displayed_fields_ids = self.displayed_fields_ids(&rtxn).unwrap(); - - let attributes_to_retrieve_ids = match query.attributes_to_retrieve { - Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, - Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), - None => None, - }; - - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { - (_, Some(ids)) => ids, - (Some(ids), None) => ids, - (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + let fields_to_display = self.fields_to_display(&rtxn, query.attributes_to_retrieve, &fields_ids_map)?; let stop_words = fst::Set::default(); let highlighter = Highlighter::new(&stop_words); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); + let mut object = milli::obkv_to_json(&fields_to_display, &fields_ids_map, obkv).unwrap(); if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } From 55fadd7f87ab6953f3f573d59c730d4966a39449 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 13:53:50 +0100 Subject: [PATCH 174/522] change facetedAttributes to attributesForFaceting --- meilisearch-http/src/index/mod.rs | 2 +- meilisearch-http/src/index/updates.rs | 6 +++--- meilisearch-http/src/routes/settings/mod.rs | 4 ++-- meilisearch-http/tests/settings/get_settings.rs | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 06f1140a3..72453091f 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -54,7 +54,7 @@ impl Index { Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), searchable_attributes: Some(Some(searchable_attributes)), - faceted_attributes: Some(Some(faceted_attributes)), + attributes_for_faceting: Some(Some(faceted_attributes)), ranking_rules: Some(Some(criteria)), }) } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 70c2dfc2b..254f6e991 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -35,7 +35,7 @@ pub struct Settings { pub searchable_attributes: Option>>, #[serde(default)] - pub faceted_attributes: Option>>, + pub attributes_for_faceting: Option>>, #[serde( default, @@ -50,7 +50,7 @@ impl Settings { Self { displayed_attributes: Some(None), searchable_attributes: Some(None), - faceted_attributes: Some(None), + attributes_for_faceting: Some(None), ranking_rules: Some(None), } } @@ -158,7 +158,7 @@ impl Index { } // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { + if let Some(ref facet_types) = settings.attributes_for_faceting { let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); builder.set_faceted_fields(facet_types); } diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 4e7219b1c..345a86534 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -82,7 +82,7 @@ macro_rules! make_setting_route { make_setting_route!( "/indexes/{index_uid}/settings/attributes-for-faceting", std::collections::HashMap, - faceted_attributes + attributes_for_faceting ); make_setting_route!( @@ -126,7 +126,7 @@ macro_rules! create_services { } create_services!( - faceted_attributes, + attributes_for_faceting, displayed_attributes, searchable_attributes ); diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index e00840f9b..0a8972108 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -19,7 +19,8 @@ async fn get_settings() { assert_eq!(settings.keys().len(), 4); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); - assert_eq!(settings["facetedAttributes"], json!({})); + println!("{:?}", settings); + assert_eq!(settings["attributesForFaceting"], json!({})); assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximity", "attribute", "wordsPosition", "exactness"])); } From f727dcc8c6d8ebcd7d0f7a4becc26ec3b1fe1c3e Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 14:26:59 +0100 Subject: [PATCH 175/522] update milli --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba4e8580e..d40beeb70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1889,6 +1889,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" +source = "git+https://github.com/meilisearch/milli.git?rev=b7b23cd#b7b23cd4a8e62932c66c2ebedf9d89ddf089e299" dependencies = [ "anyhow", "bstr", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e88556f0a..5fb5d0ebf 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -17,7 +17,6 @@ actix-cors = { path = "../../actix-extras/actix-cors" } actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } actix-service = "2.0.0-beta.4" actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } -#actix-web = { version = "3", features = ["rustls"] } anyhow = "1.0.36" async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } @@ -38,8 +37,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { path = "../../milli/milli" } -#milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "b7b23cd" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" From 0c80d891c0b953a3154de77567dea7c33f453945 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 14:29:30 +0100 Subject: [PATCH 176/522] clean Cargo.toml --- meilisearch-http/Cargo.toml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 5fb5d0ebf..9e148c46b 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -19,10 +19,14 @@ actix-service = "2.0.0-beta.4" actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } anyhow = "1.0.36" async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } +async-stream = "0.3.0" +async-trait = "0.1.42" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } bytes = "0.6.0" chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.0" +dashmap = "4.0.2" +either = "1.6.1" env_logger = "0.8.2" flate2 = "1.0.19" fst = "0.4.5" @@ -32,6 +36,7 @@ grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } heed = "0.10.6" http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } +itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } @@ -40,6 +45,7 @@ memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", rev = "b7b23cd" } mime = "0.3.16" once_cell = "1.5.2" +parking_lot = "0.11.1" rand = "0.7.3" rayon = "1.5.0" regex = "1.4.2" @@ -52,19 +58,22 @@ slice-group-by = "0.2.6" structopt = "0.3.20" tar = "0.4.29" tempfile = "3.1.0" -tokio = { version = "1", features = ["full"] } -dashmap = "4.0.2" -uuid = "0.8.2" -itertools = "0.10.0" -either = "1.6.1" -async-trait = "0.1.42" thiserror = "1.0.24" -async-stream = "0.3.0" -parking_lot = "0.11.1" +tokio = { version = "1", features = ["full"] } +uuid = "0.8.2" [dependencies.sentry] default-features = false -features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] +features = [ + "with_client_implementation", + "with_panic", + "with_failure", + "with_device_info", + "with_rust_info", + "with_reqwest_transport", + "with_rustls", + "with_env_logger" +] optional = true version = "0.18.1" From 01479dcf996d5cdcee30cbbf8d5bf912c2640cab Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 14:43:47 +0100 Subject: [PATCH 177/522] rename name to uid in code --- meilisearch-http/src/data/mod.rs | 12 ++-- meilisearch-http/src/data/updates.rs | 8 +-- .../src/index_controller/uuid_resolver.rs | 64 +++++++++---------- meilisearch-http/src/routes/index.rs | 4 +- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 176422efd..5d85650d0 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -78,21 +78,21 @@ impl Data { Ok(Data { inner }) } - pub async fn settings>(&self, index_uid: S) -> anyhow::Result { - self.index_controller.settings(index_uid.as_ref().to_string()).await + pub async fn settings>(&self, uid: S) -> anyhow::Result { + self.index_controller.settings(uid.as_ref().to_string()).await } pub async fn list_indexes(&self) -> anyhow::Result> { self.index_controller.list_indexes().await } - pub async fn index(&self, name: impl AsRef) -> anyhow::Result> { - self.index_controller.get_index(name.as_ref().to_string()).await + pub async fn index(&self, uid: impl AsRef) -> anyhow::Result> { + self.index_controller.get_index(uid.as_ref().to_string()).await } - pub async fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { + pub async fn create_index(&self, uid: impl AsRef, primary_key: Option>) -> anyhow::Result { let settings = IndexSettings { - uid: Some(name.as_ref().to_string()), + uid: Some(uid.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), }; diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index 0e2c56e42..ef9471fb5 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -65,15 +65,15 @@ impl Data { pub async fn update_index( &self, - name: impl AsRef, + uid: impl AsRef, primary_key: Option>, - new_name: Option> + new_uid: Option> ) -> anyhow::Result { let settings = IndexSettings { - uid: new_name.map(|s| s.as_ref().to_string()), + uid: new_uid.map(|s| s.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), }; - self.index_controller.update_index(name.as_ref().to_string(), settings).await + self.index_controller.update_index(uid.as_ref().to_string(), settings).await } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index fcf417248..b113de37a 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -11,19 +11,19 @@ pub type Result = std::result::Result; #[derive(Debug)] enum UuidResolveMsg { Resolve { - name: String, + uid: String, ret: oneshot::Sender>, }, GetOrCreate { - name: String, + uid: String, ret: oneshot::Sender>, }, Create { - name: String, + uid: String, ret: oneshot::Sender>, }, Delete { - name: String, + uid: String, ret: oneshot::Sender>, }, List { @@ -48,16 +48,16 @@ impl UuidResolverActor { loop { match self.inbox.recv().await { - Some(Create { name, ret }) => { + Some(Create { uid: name, ret }) => { let _ = ret.send(self.handle_create(name).await); } - Some(GetOrCreate { name, ret }) => { + Some(GetOrCreate { uid: name, ret }) => { let _ = ret.send(self.handle_get_or_create(name).await); } - Some(Resolve { name, ret }) => { + Some(Resolve { uid: name, ret }) => { let _ = ret.send(self.handle_resolve(name).await); } - Some(Delete { name, ret }) => { + Some(Delete { uid: name, ret }) => { let _ = ret.send(self.handle_delete(name).await); } Some(List { ret }) => { @@ -71,32 +71,32 @@ impl UuidResolverActor { warn!("exiting uuid resolver loop"); } - async fn handle_create(&self, name: String) -> Result { - if !is_index_uid_valid(&name) { - return Err(UuidError::BadlyFormatted(name)) + async fn handle_create(&self, uid: String) -> Result { + if !is_index_uid_valid(&uid) { + return Err(UuidError::BadlyFormatted(uid)) } - self.store.create_uuid(name, true).await + self.store.create_uuid(uid, true).await } - async fn handle_get_or_create(&self, name: String) -> Result { - if !is_index_uid_valid(&name) { - return Err(UuidError::BadlyFormatted(name)) + async fn handle_get_or_create(&self, uid: String) -> Result { + if !is_index_uid_valid(&uid) { + return Err(UuidError::BadlyFormatted(uid)) } - self.store.create_uuid(name, false).await + self.store.create_uuid(uid, false).await } - async fn handle_resolve(&self, name: String) -> Result { + async fn handle_resolve(&self, uid: String) -> Result { self.store - .get_uuid(name.clone()) + .get_uuid(uid.clone()) .await? - .ok_or(UuidError::UnexistingIndex(name)) + .ok_or(UuidError::UnexistingIndex(uid)) } - async fn handle_delete(&self, name: String) -> Result { + async fn handle_delete(&self, uid: String) -> Result { self.store - .delete(name.clone()) + .delete(uid.clone()) .await? - .ok_or(UuidError::UnexistingIndex(name)) + .ok_or(UuidError::UnexistingIndex(uid)) } async fn handle_list(&self) -> Result> { @@ -125,7 +125,7 @@ impl UuidResolverHandle { pub async fn resolve(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Resolve { name, ret }; + let msg = UuidResolveMsg::Resolve { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -134,7 +134,7 @@ impl UuidResolverHandle { pub async fn get_or_create(&self, name: String) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::GetOrCreate { name, ret }; + let msg = UuidResolveMsg::GetOrCreate { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -143,7 +143,7 @@ impl UuidResolverHandle { pub async fn create(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Create { name, ret }; + let msg = UuidResolveMsg::Create { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -152,7 +152,7 @@ impl UuidResolverHandle { pub async fn delete(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Delete { name, ret }; + let msg = UuidResolveMsg::Delete { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -189,9 +189,9 @@ pub enum UuidError { trait UuidStore { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return // the uuid otherwise. - async fn create_uuid(&self, name: String, err: bool) -> Result; - async fn get_uuid(&self, name: String) -> Result>; - async fn delete(&self, name: String) -> Result>; + async fn create_uuid(&self, uid: String, err: bool) -> Result; + async fn get_uuid(&self, uid: String) -> Result>; + async fn delete(&self, uid: String) -> Result>; async fn list(&self) -> Result>; } @@ -253,15 +253,15 @@ impl UuidStore for HeedUuidStore { }).await? } - async fn delete(&self, name: String) -> Result> { + async fn delete(&self, uid: String) -> Result> { let env = self.env.clone(); let db = self.db.clone(); tokio::task::spawn_blocking(move || { let mut txn = env.write_txn()?; - match db.get(&txn, &name)? { + match db.get(&txn, &uid)? { Some(uuid) => { let uuid = Uuid::from_slice(uuid)?; - db.delete(&mut txn, &name)?; + db.delete(&mut txn, &uid)?; txn.commit()?; Ok(Some(uuid)) } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 68c94798a..1cdfa9880 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -75,7 +75,7 @@ async fn create_index( #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateIndexRequest { - name: Option, + uid: Option, primary_key: Option, } @@ -95,7 +95,7 @@ async fn update_index( path: web::Path, body: web::Json, ) -> Result { - match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.name.as_ref()).await { + match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.uid.as_ref()).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) From abbea5973207e38ae698e347033ebba6bedf4896 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 16:52:05 +0100 Subject: [PATCH 178/522] fix clippy warnings --- meilisearch-http/src/data/mod.rs | 14 +++---- meilisearch-http/src/data/search.rs | 12 +++--- meilisearch-http/src/data/updates.rs | 40 +++++++++---------- meilisearch-http/src/index/search.rs | 2 +- meilisearch-http/src/index/updates.rs | 28 +++++-------- .../src/index_controller/index_actor.rs | 38 +++++++++--------- meilisearch-http/src/index_controller/mod.rs | 23 +++++------ .../src/index_controller/update_actor.rs | 38 +++++++++--------- .../src/index_controller/update_store.rs | 4 +- .../src/index_controller/uuid_resolver.rs | 8 ++-- meilisearch-http/src/routes/document.rs | 2 +- meilisearch-http/src/routes/index.rs | 24 +++++------ meilisearch-http/src/routes/search.rs | 12 +++--- meilisearch-http/src/routes/settings/mod.rs | 4 +- meilisearch-http/tests/index/create_index.rs | 2 +- meilisearch-http/tests/index/delete_index.rs | 2 +- meilisearch-http/tests/index/get_index.rs | 2 +- 17 files changed, 124 insertions(+), 131 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 5d85650d0..e0ae77ffb 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -78,22 +78,22 @@ impl Data { Ok(Data { inner }) } - pub async fn settings>(&self, uid: S) -> anyhow::Result { - self.index_controller.settings(uid.as_ref().to_string()).await + pub async fn settings(&self, uid: String) -> anyhow::Result { + self.index_controller.settings(uid).await } pub async fn list_indexes(&self) -> anyhow::Result> { self.index_controller.list_indexes().await } - pub async fn index(&self, uid: impl AsRef) -> anyhow::Result> { - self.index_controller.get_index(uid.as_ref().to_string()).await + pub async fn index(&self, uid: String) -> anyhow::Result { + self.index_controller.get_index(uid).await } - pub async fn create_index(&self, uid: impl AsRef, primary_key: Option>) -> anyhow::Result { + pub async fn create_index(&self, uid: String, primary_key: Option) -> anyhow::Result { let settings = IndexSettings { - uid: Some(uid.as_ref().to_string()), - primary_key: primary_key.map(|s| s.as_ref().to_string()), + uid: Some(uid), + primary_key, }; let meta = self.index_controller.create_index(settings).await?; diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index 753083b7e..8838a11e0 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -4,12 +4,12 @@ use crate::index::{SearchQuery, SearchResult}; use super::Data; impl Data { - pub async fn search>( + pub async fn search( &self, - index: S, + index: String, search_query: SearchQuery, ) -> anyhow::Result { - self.index_controller.search(index.as_ref().to_string(), search_query).await + self.index_controller.search(index, search_query).await } pub async fn retrieve_documents( @@ -24,11 +24,11 @@ impl Data { pub async fn retrieve_document( &self, - index: impl AsRef + Sync + Send + 'static, - document_id: impl AsRef + Sync + Send + 'static, + index: String, + document_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result> { - self.index_controller.document(index.as_ref().to_string(), document_id.as_ref().to_string(), attributes_to_retrieve).await + self.index_controller.document(index, document_id, attributes_to_retrieve).await } } diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index ef9471fb5..d5a246223 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -9,14 +9,14 @@ use super::Data; impl Data { pub async fn add_documents( &self, - index: impl AsRef + Send + Sync + 'static, + index: String, method: IndexDocumentsMethod, format: UpdateFormat, stream: Payload, primary_key: Option, ) -> anyhow::Result { - let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, stream, primary_key).await?; + let update_status = self.index_controller.add_documents(index, method, format, stream, primary_key).await?; Ok(update_status) } @@ -27,53 +27,53 @@ impl Data { create: bool, ) -> anyhow::Result { let update = self.index_controller.update_settings(index, settings, create).await?; - Ok(update.into()) + Ok(update) } pub async fn clear_documents( &self, - index: impl AsRef + Sync + Send + 'static, + index: String, ) -> anyhow::Result { - let update = self.index_controller.clear_documents(index.as_ref().to_string()).await?; + let update = self.index_controller.clear_documents(index).await?; Ok(update) } pub async fn delete_documents( &self, - index: impl AsRef + Sync + Send + 'static, + index: String, document_ids: Vec, ) -> anyhow::Result { - let update = self.index_controller.delete_documents(index.as_ref().to_string(), document_ids).await?; - Ok(update.into()) + let update = self.index_controller.delete_documents(index, document_ids).await?; + Ok(update) } pub async fn delete_index( &self, - index: impl AsRef + Send + Sync + 'static, + index: String, ) -> anyhow::Result<()> { - self.index_controller.delete_index(index.as_ref().to_owned()).await?; + self.index_controller.delete_index(index).await?; Ok(()) } - pub async fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { - self.index_controller.update_status(index.as_ref().to_string(), uid).await + pub async fn get_update_status(&self, index: String, uid: u64) -> anyhow::Result { + self.index_controller.update_status(index, uid).await } - pub async fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { - self.index_controller.all_update_status(index.as_ref().to_string()).await + pub async fn get_updates_status(&self, index: String) -> anyhow::Result> { + self.index_controller.all_update_status(index).await } pub async fn update_index( &self, - uid: impl AsRef, - primary_key: Option>, - new_uid: Option> + uid: String, + primary_key: Option, + new_uid: Option ) -> anyhow::Result { let settings = IndexSettings { - uid: new_uid.map(|s| s.as_ref().to_string()), - primary_key: primary_key.map(|s| s.as_ref().to_string()), + uid: new_uid, + primary_key, }; - self.index_controller.update_index(uid.as_ref().to_string(), settings).await + self.index_controller.update_index(uid, settings).await } } diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index aa2fe0715..c7db9c313 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -120,7 +120,7 @@ impl Index { fn parse_facets_array( txn: &RoTxn, index: &Index, - arr: &Vec, + arr: &[Value], ) -> anyhow::Result> { let mut ands = Vec::new(); for value in arr { diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 254f6e991..b2d95c9ca 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -85,11 +85,8 @@ impl Index { let mut wtxn = self.write_txn()?; // Set the primary key if not set already, ignore if already set. - match (self.primary_key(&wtxn)?, primary_key) { - (None, Some(ref primary_key)) => { - self.put_primary_key(&mut wtxn, primary_key)?; - } - _ => (), + if let (None, Some(ref primary_key)) = (self.primary_key(&wtxn)?, primary_key) { + self.put_primary_key(&mut wtxn, primary_key)?; } let mut builder = update_builder.index_documents(&mut wtxn, self); @@ -109,13 +106,10 @@ impl Index { info!("document addition done: {:?}", result); - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()), - } + result.and_then(|addition_result| wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into)) } pub fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { @@ -128,7 +122,7 @@ impl Index { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()), + Err(e) => Err(e), } } @@ -159,7 +153,7 @@ impl Index { // We transpose the settings JSON struct into a real setting update. if let Some(ref facet_types) = settings.attributes_for_faceting { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + let facet_types = facet_types.clone().unwrap_or_else(HashMap::new); builder.set_faceted_fields(facet_types); } @@ -179,7 +173,7 @@ impl Index { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()), + Err(e) => Err(e), } } @@ -202,7 +196,7 @@ impl Index { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()), + Err(e) => Err(e), } } @@ -223,7 +217,7 @@ impl Index { .commit() .and(Ok(UpdateResult::DocumentDeletion { deleted })) .map_err(Into::into), - Err(e) => Err(e.into()) + Err(e) => Err(e) } } } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index ee7a9e782..605dee22b 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -92,7 +92,7 @@ enum IndexMsg { }, GetMeta { uuid: Uuid, - ret: oneshot::Sender>>, + ret: oneshot::Sender>, }, UpdateIndex { uuid: Uuid, @@ -137,7 +137,7 @@ impl IndexActor { ) -> Result { let options = IndexerOpts::default(); let update_handler = - UpdateHandler::new(&options).map_err(|e| IndexError::Error(e.into()))?; + UpdateHandler::new(&options).map_err(IndexError::Error)?; let update_handler = Arc::new(update_handler); let read_receiver = Some(read_receiver); let write_receiver = Some(write_receiver); @@ -274,11 +274,11 @@ impl IndexActor { data: File, ) -> Result { debug!("Processing update {}", meta.id()); - let uuid = meta.index_uuid().clone(); + let uuid = meta.index_uuid(); let update_handler = self.update_handler.clone(); - let index = match self.store.get(uuid.clone()).await? { + let index = match self.store.get(*uuid).await? { Some(index) => index, - None => self.store.create(uuid, None).await?, + None => self.store.create(*uuid, None).await?, }; spawn_blocking(move || update_handler.handle_update(meta, data, index)) .await @@ -291,7 +291,7 @@ impl IndexActor { .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || index.settings().map_err(|e| IndexError::Error(e))) + spawn_blocking(move || index.settings().map_err(IndexError::Error)) .await .map_err(|e| IndexError::Error(e.into()))? } @@ -311,7 +311,7 @@ impl IndexActor { spawn_blocking(move || { index .retrieve_documents(offset, limit, attributes_to_retrieve) - .map_err(|e| IndexError::Error(e)) + .map_err(IndexError::Error) }) .await .map_err(|e| IndexError::Error(e.into()))? @@ -331,7 +331,7 @@ impl IndexActor { spawn_blocking(move || { index .retrieve_document(doc_id, attributes_to_retrieve) - .map_err(|e| IndexError::Error(e)) + .map_err(IndexError::Error) }) .await .map_err(|e| IndexError::Error(e.into()))? @@ -354,15 +354,15 @@ impl IndexActor { Ok(()) } - async fn handle_get_meta(&self, uuid: Uuid) -> Result> { + async fn handle_get_meta(&self, uuid: Uuid) -> Result { match self.store.get(uuid).await? { Some(index) => { let meta = spawn_blocking(move || IndexMeta::new(&index)) .await .map_err(|e| IndexError::Error(e.into()))??; - Ok(Some(meta)) + Ok(meta) } - None => Ok(None), + None => Err(IndexError::UnexistingIndex), } } @@ -405,7 +405,7 @@ impl IndexActorHandle { let (read_sender, read_receiver) = mpsc::channel(100); let (write_sender, write_receiver) = mpsc::channel(100); - let store = HeedIndexStore::new(path, index_size)?; + let store = HeedIndexStore::new(path, index_size); let actor = IndexActor::new(read_receiver, write_receiver, store)?; tokio::task::spawn(actor.run()); Ok(Self { @@ -492,7 +492,7 @@ impl IndexActorHandle { Ok(receiver.await.expect("IndexActor has been killed")?) } - pub async fn get_index_meta(&self, uuid: Uuid) -> Result> { + pub async fn get_index_meta(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetMeta { uuid, ret }; let _ = self.read_sender.send(msg).await; @@ -518,14 +518,14 @@ struct HeedIndexStore { } impl HeedIndexStore { - fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { + fn new(path: impl AsRef, index_size: usize) -> Self { let path = path.as_ref().join("indexes/"); let index_store = Arc::new(RwLock::new(HashMap::new())); - Ok(Self { + Self { index_store, path, index_size, - }) + } } } @@ -550,7 +550,7 @@ impl IndexStore for HeedIndexStore { .await .map_err(|e| IndexError::Error(e.into()))??; - self.index_store.write().await.insert(uuid.clone(), index.clone()); + self.index_store.write().await.insert(uuid, index.clone()); Ok(index) } @@ -574,7 +574,7 @@ impl IndexStore for HeedIndexStore { self.index_store .write() .await - .insert(uuid.clone(), index.clone()); + .insert(uuid, index.clone()); Ok(Some(index)) } } @@ -595,6 +595,6 @@ fn open_index(path: impl AsRef, size: usize) -> Result { let mut options = EnvOpenOptions::new(); options.map_size(size); let index = milli::Index::new(options, &path) - .map_err(|e| IndexError::Error(e))?; + .map_err(IndexError::Error)?; Ok(Index(Arc::new(index))) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index d82b304a2..ef8b16340 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -127,7 +127,7 @@ impl IndexController { // the update is processed. This would make calls to GET index to fail until the update // is complete. Since this is get or create, we ignore the error when the index already // exists. - match self.index_handle.create_index(uuid.clone(), None).await { + match self.index_handle.create_index(uuid, None).await { Ok(_) | Err(index_actor::IndexError::IndexAlreadyExists) => (), Err(e) => return Err(e.into()), } @@ -158,12 +158,12 @@ impl IndexController { let uuid = self.uuid_resolver .delete(uid) .await?; - self.update_handle.delete(uuid.clone()).await?; + self.update_handle.delete(uuid).await?; self.index_handle.delete(uuid).await?; Ok(()) } - pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result> { + pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result { let uuid = self.uuid_resolver .resolve(uid) .await?; @@ -184,10 +184,9 @@ impl IndexController { let mut ret = Vec::new(); for (uid, uuid) in uuids { - if let Some(meta) = self.index_handle.get_index_meta(uuid).await? { - let meta = IndexMetadata { uid, meta }; - ret.push(meta); - } + let meta = self.index_handle.get_index_meta(uuid).await?; + let meta = IndexMetadata { uid, meta }; + ret.push(meta); } Ok(ret) @@ -247,13 +246,13 @@ impl IndexController { Ok(result) } - pub async fn get_index(&self, uid: String) -> anyhow::Result> { + pub async fn get_index(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(uid.clone()).await?; - let result = self.index_handle + let meta = self.index_handle .get_index_meta(uuid) - .await? - .map(|meta| IndexMetadata { uid, meta }); - Ok(result) + .await?; + let meta = IndexMetadata { uid, meta }; + Ok(meta) } } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 2f66e8d20..2b9d05a4b 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -25,6 +25,8 @@ pub enum UpdateError { Error(Box), #[error("Index {0} doesn't exist.")] UnexistingIndex(Uuid), + #[error("Update {0} doesn't exist.")] + UnexistingUpdate(u64), } enum UpdateMsg { @@ -40,7 +42,7 @@ enum UpdateMsg { }, GetUpdate { uuid: Uuid, - ret: oneshot::Sender>>, + ret: oneshot::Sender>, id: u64, }, Delete { @@ -62,8 +64,8 @@ struct UpdateActor { #[async_trait::async_trait] trait UpdateStoreStore { async fn get_or_create(&self, uuid: Uuid) -> Result>; - async fn delete(&self, uuid: &Uuid) -> Result>>; - async fn get(&self, uuid: &Uuid) -> Result>>; + async fn delete(&self, uuid: Uuid) -> Result>>; + async fn get(&self, uuid: Uuid) -> Result>>; } impl UpdateActor @@ -145,18 +147,17 @@ where .map_err(|e| UpdateError::Error(Box::new(e)))?; tokio::task::spawn_blocking(move || { - let result = update_store + update_store .register_update(meta, path, uuid) - .map(|pending| UpdateStatus::Pending(pending)) - .map_err(|e| UpdateError::Error(Box::new(e))); - result + .map(UpdateStatus::Pending) + .map_err(|e| UpdateError::Error(Box::new(e))) }) .await .map_err(|e| UpdateError::Error(Box::new(e)))? } async fn handle_list_updates(&self, uuid: Uuid) -> Result> { - let update_store = self.store.get(&uuid).await?; + let update_store = self.store.get(uuid).await?; tokio::task::spawn_blocking(move || { let result = update_store .ok_or(UpdateError::UnexistingIndex(uuid))? @@ -168,20 +169,21 @@ where .map_err(|e| UpdateError::Error(Box::new(e)))? } - async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result> { + async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { let store = self .store - .get(&uuid) + .get(uuid) .await? .ok_or(UpdateError::UnexistingIndex(uuid))?; let result = store .meta(id) - .map_err(|e| UpdateError::Error(Box::new(e)))?; + .map_err(|e| UpdateError::Error(Box::new(e)))? + .ok_or(UpdateError::UnexistingUpdate(id))?; Ok(result) } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let store = self.store.delete(&uuid).await?; + let store = self.store.delete(uuid).await?; if let Some(store) = store { tokio::task::spawn(async move { @@ -246,7 +248,7 @@ where receiver.await.expect("update actor killed.") } - pub async fn update_status(&self, uuid: Uuid, id: u64) -> Result> { + pub async fn update_status(&self, uuid: Uuid, id: u64) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::GetUpdate { uuid, id, ret }; let _ = self.sender.send(msg).await; @@ -310,9 +312,9 @@ impl UpdateStoreStore for MapUpdateStoreStore { } } - async fn get(&self, uuid: &Uuid) -> Result>> { + async fn get(&self, uuid: Uuid) -> Result>> { let guard = self.db.read().await; - match guard.get(uuid) { + match guard.get(&uuid) { Some(uuid) => Ok(Some(uuid.clone())), None => { // The index is not found in the found in the loaded indexes, so we attempt to load @@ -322,7 +324,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { let path = self.path.clone().join(format!("updates-{}", uuid)); if path.exists() { let mut guard = self.db.write().await; - match guard.entry(uuid.clone()) { + match guard.entry(uuid) { Entry::Vacant(entry) => { // We can safely load the index let index_handle = self.index_handle.clone(); @@ -333,7 +335,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { futures::executor::block_on(index_handle.update(meta, file)) }) .map_err(|e| UpdateError::Error(e.into()))?; - let store = entry.insert(store.clone()); + let store = entry.insert(store); Ok(Some(store.clone())) } Entry::Occupied(entry) => { @@ -348,7 +350,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { } } - async fn delete(&self, uuid: &Uuid) -> Result>> { + async fn delete(&self, uuid: Uuid) -> Result>> { let store = self.db.write().await.remove(&uuid); let path = self.path.clone().join(format!("updates-{}", uuid)); if store.is_some() || path.exists() { diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index a7a65e494..9ce2f242b 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -92,7 +92,7 @@ where let update_store_weak = Arc::downgrade(&update_store); tokio::task::spawn(async move { // Block and wait for something to process. - 'outer: while let Some(_) = notification_receiver.recv().await { + 'outer: while notification_receiver.recv().await.is_some() { loop { match update_store_weak.upgrade() { Some(update_store) => { @@ -276,7 +276,7 @@ where updates.extend(failed); - updates.sort_unstable_by(|a, b| a.id().cmp(&b.id())); + updates.sort_by_key(|u| u.id()); Ok(updates) } diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index b113de37a..69e9a626b 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -216,7 +216,7 @@ impl HeedUuidStore { impl UuidStore for HeedUuidStore { async fn create_uuid(&self, name: String, err: bool) -> Result { let env = self.env.clone(); - let db = self.db.clone(); + let db = self.db; tokio::task::spawn_blocking(move || { let mut txn = env.write_txn()?; match db.get(&txn, &name)? { @@ -240,7 +240,7 @@ impl UuidStore for HeedUuidStore { async fn get_uuid(&self, name: String) -> Result> { let env = self.env.clone(); - let db = self.db.clone(); + let db = self.db; tokio::task::spawn_blocking(move || { let txn = env.read_txn()?; match db.get(&txn, &name)? { @@ -255,7 +255,7 @@ impl UuidStore for HeedUuidStore { async fn delete(&self, uid: String) -> Result> { let env = self.env.clone(); - let db = self.db.clone(); + let db = self.db; tokio::task::spawn_blocking(move || { let mut txn = env.write_txn()?; match db.get(&txn, &uid)? { @@ -272,7 +272,7 @@ impl UuidStore for HeedUuidStore { async fn list(&self) -> Result> { let env = self.env.clone(); - let db = self.db.clone(); + let db = self.db; tokio::task::spawn_blocking(move || { let txn = env.read_txn()?; let mut entries = Vec::new(); diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 43d88e4b9..c53c3cb85 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -105,7 +105,7 @@ async fn get_all_documents( .attributes_to_retrieve .as_ref() .map(|attrs| attrs - .split(",") + .split(',') .map(String::from) .collect::>()); diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 1cdfa9880..cb1c08e24 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -37,13 +37,12 @@ async fn get_index( data: web::Data, path: web::Path, ) -> Result { - match data.index(&path.index_uid).await? { - Some(meta) => { + match data.index(path.index_uid.clone()).await { + Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) } - None => { - let e = format!("Index {:?} doesn't exist.", path.index_uid); + Err(e) => { Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } @@ -61,7 +60,8 @@ async fn create_index( data: web::Data, body: web::Json, ) -> Result { - match data.create_index(&body.uid, body.primary_key.clone()).await { + let body = body.into_inner(); + match data.create_index(body.uid, body.primary_key).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -95,7 +95,8 @@ async fn update_index( path: web::Path, body: web::Json, ) -> Result { - match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.uid.as_ref()).await { + let body = body.into_inner(); + match data.update_index(path.into_inner().index_uid, body.primary_key, body.uid).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -133,16 +134,13 @@ async fn get_update_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_update_status(&path.index_uid, path.update_id).await; + let params = path.into_inner(); + let result = data.get_update_status(params.index_uid, params.update_id).await; match result { - Ok(Some(meta)) => { + Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) } - Ok(None) => { - let e = format!("update {} for index {:?} doesn't exists.", path.update_id, path.index_uid); - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } Err(e) => { Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } @@ -154,7 +152,7 @@ async fn get_all_updates_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_updates_status(&path.index_uid).await; + let result = data.get_updates_status(path.into_inner().index_uid).await; match result { Ok(metas) => { let json = serde_json::to_string(&metas).unwrap(); diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index eb03cf94d..2e718ebe9 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -36,19 +36,19 @@ impl TryFrom for SearchQuery { fn try_from(other: SearchQueryGet) -> anyhow::Result { let attributes_to_retrieve = other .attributes_to_retrieve - .map(|attrs| attrs.split(",").map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let attributes_to_crop = other .attributes_to_crop - .map(|attrs| attrs.split(",").map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let attributes_to_highlight = other .attributes_to_highlight - .map(|attrs| attrs.split(",").map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let facet_distributions = other .facet_distributions - .map(|attrs| attrs.split(",").map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let facet_filters = match other.facet_filters { Some(ref f) => Some(serde_json::from_str(f)?), @@ -83,7 +83,7 @@ async fn search_with_url_query( return Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } }; - let search_result = data.search(&path.index_uid, query).await; + let search_result = data.search(path.into_inner().index_uid, query).await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); @@ -101,7 +101,7 @@ async fn search_with_post( path: web::Path, params: web::Json, ) -> Result { - let search_result = data.search(&path.index_uid, params.into_inner()).await; + let search_result = data.search(path.into_inner().index_uid, params.into_inner()).await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 345a86534..df2fe9f15 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -64,7 +64,7 @@ macro_rules! make_setting_route { data: actix_web::web::Data, index_uid: actix_web::web::Path, ) -> std::result::Result { - match data.settings(index_uid.as_ref()).await { + match data.settings(index_uid.into_inner()).await { Ok(settings) => { let setting = settings.$attr; let json = serde_json::to_string(&setting).unwrap(); @@ -153,7 +153,7 @@ async fn get_all( data: web::Data, index_uid: web::Path, ) -> Result { - match data.settings(index_uid.as_ref()).await { + match data.settings(index_uid.into_inner()).await { Ok(settings) => { let json = serde_json::to_string(&settings).unwrap(); Ok(HttpResponse::Ok().body(json)) diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 0bf2f15a5..8f7224602 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -69,5 +69,5 @@ async fn test_create_multiple_indexes() { assert_eq!(index1.get().await.1, 200); assert_eq!(index2.get().await.1, 200); assert_eq!(index3.get().await.1, 200); - assert_eq!(index4.get().await.1, 404); + assert_eq!(index4.get().await.1, 400); } diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 5bc78950f..9d145899e 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -14,7 +14,7 @@ async fn create_and_delete_index() { assert_eq!(code, 200); - assert_eq!(index.get().await.1, 404); + assert_eq!(index.get().await.1, 400); } #[actix_rt::test] diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index a1a95eef8..1a1815dfe 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -29,7 +29,7 @@ async fn get_unexisting_index() { let (_response, code) = index.get().await; - assert_eq!(code, 404); + assert_eq!(code, 400); } #[actix_rt::test] From c29b86849be93857056fbd0be9529d8011b15827 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 17:40:20 +0100 Subject: [PATCH 179/522] use actix cors git dependency --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index d40beeb70..bcdcec71c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,6 +54,7 @@ dependencies = [ [[package]] name = "actix-cors" version = "0.5.4" +source = "git+https://github.com/MarinPostma/actix-extras.git?rev=8f7b1fd#8f7b1fdd7f58f3a15542a27e6b237666bac756d4" dependencies = [ "actix-web", "derive_more", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 9e148c46b..9ae7baae2 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -13,7 +13,7 @@ path = "src/main.rs" vergen = "3.1.0" [dependencies] -actix-cors = { path = "../../actix-extras/actix-cors" } +actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "8f7b1fd" } actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } actix-service = "2.0.0-beta.4" actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } From dd324807f92db888561a5c80c57336c671c984c3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 18:11:10 +0100 Subject: [PATCH 180/522] last review edits + fmt --- meilisearch-error/src/lib.rs | 33 +++- meilisearch-http/src/data/mod.rs | 16 +- meilisearch-http/src/data/search.rs | 13 +- meilisearch-http/src/data/updates.rs | 37 ++-- meilisearch-http/src/error.rs | 47 +++-- .../src/helpers/authentication.rs | 17 +- meilisearch-http/src/helpers/compression.rs | 4 +- meilisearch-http/src/helpers/mod.rs | 2 - .../src/helpers/normalize_path.rs | 86 -------- meilisearch-http/src/index/mod.rs | 19 +- meilisearch-http/src/index/search.rs | 35 ++-- meilisearch-http/src/index/updates.rs | 32 +-- .../src/index_controller/index_actor.rs | 88 +++++---- meilisearch-http/src/index_controller/mod.rs | 104 ++++++---- .../src/index_controller/update_actor.rs | 14 +- .../src/index_controller/update_handler.rs | 9 +- .../src/index_controller/update_store.rs | 185 +++++++++--------- .../src/index_controller/updates.rs | 4 +- .../src/index_controller/uuid_resolver.rs | 26 ++- meilisearch-http/src/lib.rs | 88 ++++----- meilisearch-http/src/main.rs | 29 +-- meilisearch-http/src/option.rs | 10 +- meilisearch-http/src/routes/document.rs | 51 +++-- meilisearch-http/src/routes/index.rs | 12 +- meilisearch-http/src/routes/key.rs | 2 +- meilisearch-http/src/routes/search.rs | 10 +- meilisearch-http/src/routes/settings/mod.rs | 29 +-- meilisearch-http/src/routes/stats.rs | 4 +- meilisearch-http/src/routes/stop_words.rs | 2 +- meilisearch-http/src/routes/synonym.rs | 2 +- meilisearch-http/tests/common/index.rs | 26 ++- meilisearch-http/tests/common/mod.rs | 2 +- meilisearch-http/tests/common/server.rs | 7 +- meilisearch-http/tests/common/service.rs | 23 ++- .../tests/documents/add_documents.rs | 40 ++-- .../tests/documents/delete_documents.rs | 31 ++- .../tests/documents/get_documents.rs | 125 +++++++++--- meilisearch-http/tests/documents/mod.rs | 2 +- meilisearch-http/tests/index/create_index.rs | 1 - meilisearch-http/tests/index/delete_index.rs | 2 - meilisearch-http/tests/index/get_index.rs | 10 +- meilisearch-http/tests/index/mod.rs | 2 +- meilisearch-http/tests/integration.rs | 2 +- meilisearch-http/tests/search/mod.rs | 1 - .../tests/settings/get_settings.rs | 43 ++-- meilisearch-http/tests/updates/mod.rs | 26 +-- 46 files changed, 764 insertions(+), 589 deletions(-) delete mode 100644 meilisearch-http/src/helpers/normalize_path.rs diff --git a/meilisearch-error/src/lib.rs b/meilisearch-error/src/lib.rs index d0e00e9be..4cf801c72 100644 --- a/meilisearch-error/src/lib.rs +++ b/meilisearch-error/src/lib.rs @@ -81,7 +81,6 @@ pub enum Code { } impl Code { - /// ascociate a `Code` variant to the actual ErrCode fn err_code(&self) -> ErrCode { use Code::*; @@ -94,17 +93,23 @@ impl Code { // thrown when requesting an unexisting index IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND), InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST), - OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR), + OpenIndex => { + ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR) + } // invalid state error InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR), // thrown when no primary key has been set MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST), // error thrown when trying to set an already existing primary key - PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST), + PrimaryKeyAlreadyPresent => { + ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST) + } // invalid document - MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST), + MaxFieldsLimitExceeded => { + ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST) + } MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST), // error related to facets @@ -117,16 +122,26 @@ impl Code { DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND), Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR), InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN), - MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED), + MissingAuthorizationHeader => { + ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED) + } NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND), PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE), - RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST), + RetrieveDocument => { + ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST) + } SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST), - UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE), + UnsupportedMediaType => { + ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE) + } // error related to dump - DumpAlreadyInProgress => ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT), - DumpProcessFailed => ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR), + DumpAlreadyInProgress => { + ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT) + } + DumpProcessFailed => { + ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR) + } } } diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index e0ae77ffb..c312c2912 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -7,9 +7,9 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexMetadata, IndexSettings}; -use crate::index_controller::IndexController; use crate::index::Settings; +use crate::index_controller::IndexController; +use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::option::Opt; #[derive(Clone)] @@ -72,7 +72,11 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { index_controller, options, api_keys }; + let inner = DataInner { + index_controller, + options, + api_keys, + }; let inner = Arc::new(inner); Ok(Data { inner }) @@ -90,7 +94,11 @@ impl Data { self.index_controller.get_index(uid).await } - pub async fn create_index(&self, uid: String, primary_key: Option) -> anyhow::Result { + pub async fn create_index( + &self, + uid: String, + primary_key: Option, + ) -> anyhow::Result { let settings = IndexSettings { uid: Some(uid), primary_key, diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index 8838a11e0..1a998b997 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -1,7 +1,7 @@ use serde_json::{Map, Value}; -use crate::index::{SearchQuery, SearchResult}; use super::Data; +use crate::index::{SearchQuery, SearchResult}; impl Data { pub async fn search( @@ -19,7 +19,9 @@ impl Data { limit: usize, attributes_to_retrieve: Option>, ) -> anyhow::Result>> { - self.index_controller.documents(index, offset, limit, attributes_to_retrieve).await + self.index_controller + .documents(index, offset, limit, attributes_to_retrieve) + .await } pub async fn retrieve_document( @@ -27,8 +29,9 @@ impl Data { index: String, document_id: String, attributes_to_retrieve: Option>, - ) -> anyhow::Result> - { - self.index_controller.document(index, document_id, attributes_to_retrieve).await + ) -> anyhow::Result> { + self.index_controller + .document(index, document_id, attributes_to_retrieve) + .await } } diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index d5a246223..518f9fcc3 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,10 +1,9 @@ -use milli::update::{IndexDocumentsMethod, UpdateFormat}; use actix_web::web::Payload; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus}; -use crate::index::Settings; use super::Data; - +use crate::index::Settings; +use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus}; impl Data { pub async fn add_documents( @@ -14,9 +13,11 @@ impl Data { format: UpdateFormat, stream: Payload, primary_key: Option, - ) -> anyhow::Result - { - let update_status = self.index_controller.add_documents(index, method, format, stream, primary_key).await?; + ) -> anyhow::Result { + let update_status = self + .index_controller + .add_documents(index, method, format, stream, primary_key) + .await?; Ok(update_status) } @@ -26,14 +27,14 @@ impl Data { settings: Settings, create: bool, ) -> anyhow::Result { - let update = self.index_controller.update_settings(index, settings, create).await?; + let update = self + .index_controller + .update_settings(index, settings, create) + .await?; Ok(update) } - pub async fn clear_documents( - &self, - index: String, - ) -> anyhow::Result { + pub async fn clear_documents(&self, index: String) -> anyhow::Result { let update = self.index_controller.clear_documents(index).await?; Ok(update) } @@ -43,14 +44,14 @@ impl Data { index: String, document_ids: Vec, ) -> anyhow::Result { - let update = self.index_controller.delete_documents(index, document_ids).await?; + let update = self + .index_controller + .delete_documents(index, document_ids) + .await?; Ok(update) } - pub async fn delete_index( - &self, - index: String, - ) -> anyhow::Result<()> { + pub async fn delete_index(&self, index: String) -> anyhow::Result<()> { self.index_controller.delete_index(index).await?; Ok(()) } @@ -67,7 +68,7 @@ impl Data { &self, uid: String, primary_key: Option, - new_uid: Option + new_uid: Option, ) -> anyhow::Result { let settings = IndexSettings { uid: new_uid, diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 70e9c4ed8..8bfdb3573 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -1,14 +1,13 @@ use std::error; use std::fmt; -use actix_web::dev::HttpResponseBuilder; -use actix_web::http::Error as HttpError; use actix_web as aweb; +use actix_web::dev::HttpResponseBuilder; use actix_web::error::{JsonPayloadError, QueryPayloadError}; +use actix_web::http::Error as HttpError; use actix_web::http::StatusCode; -use serde::ser::{Serialize, Serializer, SerializeStruct}; -use meilisearch_error::{ErrorCode, Code}; - +use meilisearch_error::{Code, ErrorCode}; +use serde::ser::{Serialize, SerializeStruct, Serializer}; #[derive(Debug)] pub struct ResponseError { @@ -32,19 +31,25 @@ impl fmt::Display for ResponseError { // TODO: remove this when implementing actual error handling impl From for ResponseError { fn from(other: anyhow::Error) -> ResponseError { - ResponseError { inner: Box::new(Error::NotFound(other.to_string())) } + ResponseError { + inner: Box::new(Error::NotFound(other.to_string())), + } } } impl From for ResponseError { fn from(error: Error) -> ResponseError { - ResponseError { inner: Box::new(error) } + ResponseError { + inner: Box::new(error), + } } } impl From for ResponseError { fn from(err: FacetCountError) -> ResponseError { - ResponseError { inner: Box::new(err) } + ResponseError { + inner: Box::new(err), + } } } @@ -130,7 +135,10 @@ impl ErrorCode for Error { pub enum FacetCountError { AttributeNotSet(String), SyntaxError(String), - UnexpectedToken { found: String, expected: &'static [&'static str] }, + UnexpectedToken { + found: String, + expected: &'static [&'static str], + }, NoFacetSet, } @@ -143,7 +151,10 @@ impl ErrorCode for FacetCountError { } impl FacetCountError { - pub fn unexpected_token(found: impl ToString, expected: &'static [&'static str]) -> FacetCountError { + pub fn unexpected_token( + found: impl ToString, + expected: &'static [&'static str], + ) -> FacetCountError { let found = found.to_string(); FacetCountError::UnexpectedToken { expected, found } } @@ -162,7 +173,9 @@ impl fmt::Display for FacetCountError { match self { AttributeNotSet(attr) => write!(f, "Attribute {} is not set as facet", attr), SyntaxError(msg) => write!(f, "Syntax error: {}", msg), - UnexpectedToken { expected, found } => write!(f, "Unexpected {} found, expected {:?}", found, expected), + UnexpectedToken { expected, found } => { + write!(f, "Unexpected {} found, expected {:?}", found, expected) + } NoFacetSet => write!(f, "Can't perform facet count, as no facet is set"), } } @@ -276,10 +289,14 @@ impl From for Error { impl From for Error { fn from(err: JsonPayloadError) -> Error { match err { - JsonPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid JSON: {}", err)), + JsonPayloadError::Deserialize(err) => { + Error::BadRequest(format!("Invalid JSON: {}", err)) + } JsonPayloadError::Overflow => Error::PayloadTooLarge, JsonPayloadError::ContentType => Error::UnsupportedMediaType, - JsonPayloadError::Payload(err) => Error::BadRequest(format!("Problem while decoding the request: {}", err)), + JsonPayloadError::Payload(err) => { + Error::BadRequest(format!("Problem while decoding the request: {}", err)) + } } } } @@ -287,7 +304,9 @@ impl From for Error { impl From for Error { fn from(err: QueryPayloadError) -> Error { match err { - QueryPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid query parameters: {}", err)), + QueryPayloadError::Deserialize(err) => { + Error::BadRequest(format!("Invalid query parameters: {}", err)) + } } } } diff --git a/meilisearch-http/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs index 4fdbdeef5..9944c0bd4 100644 --- a/meilisearch-http/src/helpers/authentication.rs +++ b/meilisearch-http/src/helpers/authentication.rs @@ -3,7 +3,7 @@ use std::pin::Pin; use std::rc::Rc; use std::task::{Context, Poll}; -use actix_web::dev::{Transform, Service, ServiceResponse, ServiceRequest}; +use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; use actix_web::web; use futures::future::{err, ok, Future, Ready}; @@ -70,10 +70,16 @@ where let auth_header = match req.headers().get("X-Meili-API-Key") { Some(auth) => match auth.to_str() { Ok(auth) => auth, - Err(_) => return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())), + Err(_) => { + return Box::pin(err( + ResponseError::from(Error::MissingAuthorizationHeader).into() + )) + } }, None => { - return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())); + return Box::pin(err( + ResponseError::from(Error::MissingAuthorizationHeader).into() + )); } }; @@ -93,9 +99,10 @@ where if authenticated { Box::pin(svc.call(req)) } else { - Box::pin(err( - ResponseError::from(Error::InvalidToken(auth_header.to_string())).into() + Box::pin(err(ResponseError::from(Error::InvalidToken( + auth_header.to_string(), )) + .into())) } } } diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index ff3e1258f..bbf14d578 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -1,9 +1,9 @@ -use flate2::Compression; use flate2::read::GzDecoder; use flate2::write::GzEncoder; +use flate2::Compression; use std::fs::{create_dir_all, File}; use std::path::Path; -use tar::{Builder, Archive}; +use tar::{Archive, Builder}; use crate::error::Error; diff --git a/meilisearch-http/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs index 5685345b2..d1204908f 100644 --- a/meilisearch-http/src/helpers/mod.rs +++ b/meilisearch-http/src/helpers/mod.rs @@ -1,6 +1,4 @@ pub mod authentication; -//pub mod normalize_path; pub mod compression; pub use authentication::Authentication; -//pub use normalize_path::NormalizePath; diff --git a/meilisearch-http/src/helpers/normalize_path.rs b/meilisearch-http/src/helpers/normalize_path.rs deleted file mode 100644 index 51c64c52b..000000000 --- a/meilisearch-http/src/helpers/normalize_path.rs +++ /dev/null @@ -1,86 +0,0 @@ -/// From https://docs.rs/actix-web/3.0.0-alpha.2/src/actix_web/middleware/normalize.rs.html#34 -use actix_web::http::Error; -use actix_service::{Service, Transform}; -use actix_web::{ - dev::ServiceRequest, - dev::ServiceResponse, - http::uri::{PathAndQuery, Uri}, -}; -use futures::future::{ok, Ready}; -use regex::Regex; -use std::task::{Context, Poll}; -pub struct NormalizePath; - -impl Transform for NormalizePath -where - S: Service, Error = Error>, - S::Future: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type InitError = (); - type Transform = NormalizePathNormalization; - type Future = Ready>; - - fn new_transform(&self, service: S) -> Self::Future { - ok(NormalizePathNormalization { - service, - merge_slash: Regex::new("//+").unwrap(), - }) - } -} - -pub struct NormalizePathNormalization { - service: S, - merge_slash: Regex, -} - -impl Service for NormalizePathNormalization -where - S: Service, Error = Error>, - S::Future: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type Future = S::Future; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.service.poll_ready(cx) - } - - fn call(&mut self, mut req: ServiceRequest) -> Self::Future { - let head = req.head_mut(); - - // always add trailing slash, might be an extra one - let path = head.uri.path().to_string() + "/"; - - if self.merge_slash.find(&path).is_some() { - // normalize multiple /'s to one / - let path = self.merge_slash.replace_all(&path, "/"); - - let path = if path.len() > 1 { - path.trim_end_matches('/') - } else { - &path - }; - - let mut parts = head.uri.clone().into_parts(); - let pq = parts.path_and_query.as_ref().unwrap(); - - let path = if let Some(q) = pq.query() { - bytes::Bytes::from(format!("{}?{}", path, q)) - } else { - bytes::Bytes::copy_from_slice(path.as_bytes()) - }; - parts.path_and_query = Some(PathAndQuery::from_maybe_shared(path).unwrap()); - - let uri = Uri::from_parts(parts).unwrap(); - req.match_info_mut().get_mut().update(&uri); - req.head_mut().uri = uri; - } - - self.service.call(req) - } -} diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 72453091f..188afd522 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -59,19 +59,17 @@ impl Index { }) } - pub fn retrieve_documents( + pub fn retrieve_documents>( &self, offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> anyhow::Result>> - where - S: AsRef + Send + Sync + 'static, - { + ) -> anyhow::Result>> { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = + self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -95,7 +93,8 @@ impl Index { let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = + self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; let internal_id = self .external_documents_ids(&txn)? @@ -109,11 +108,7 @@ impl Index { .map(|(_, d)| d); match document { - Some(document) => Ok(obkv_to_json( - &fields_to_display, - &fields_ids_map, - document, - )?), + Some(document) => Ok(obkv_to_json(&fields_to_display, &fields_ids_map, document)?), None => bail!("Document with id {} not found", doc_id), } } diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index c7db9c313..a0de632be 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,14 +1,14 @@ -use std::time::Instant; -use std::collections::{HashSet, BTreeMap}; +use std::collections::{BTreeMap, HashSet}; use std::mem; +use std::time::Instant; -use either::Either; use anyhow::bail; +use either::Either; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{FacetCondition, MatchingWords, facet::FacetValue}; -use serde::{Serialize, Deserialize}; -use serde_json::{Value, Map}; +use milli::{facet::FacetValue, FacetCondition, MatchingWords}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; use super::Index; @@ -78,13 +78,15 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let fields_to_display = self.fields_to_display(&rtxn, query.attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = + self.fields_to_display(&rtxn, query.attributes_to_retrieve, &fields_ids_map)?; let stop_words = fst::Set::default(); let highlighter = Highlighter::new(&stop_words); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let mut object = milli::obkv_to_json(&fields_to_display, &fields_ids_map, obkv).unwrap(); + let mut object = + milli::obkv_to_json(&fields_to_display, &fields_ids_map, obkv).unwrap(); if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } @@ -183,15 +185,15 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } Value::Array(values) => Value::Array( values - .into_iter() - .map(|v| self.highlight_value(v, words_to_highlight)) - .collect(), + .into_iter() + .map(|v| self.highlight_value(v, words_to_highlight)) + .collect(), ), Value::Object(object) => Value::Object( object - .into_iter() - .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) - .collect(), + .into_iter() + .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) + .collect(), ), } } @@ -221,9 +223,6 @@ fn parse_facets( // Disabled for now //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), Value::Array(arr) => parse_facets_array(txn, index, arr), - v => bail!( - "Invalid facet expression, expected Array, found: {:?}", - v - ), + v => bail!("Invalid facet expression, expected Array, found: {:?}", v), } } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index b2d95c9ca..a519eaa82 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -4,8 +4,8 @@ use std::num::NonZeroUsize; use flate2::read::GzDecoder; use log::info; -use milli::update::{UpdateFormat, IndexDocumentsMethod, UpdateBuilder, DocumentAdditionResult}; -use serde::{Serialize, Deserialize, de::Deserializer}; +use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use serde::{de::Deserializer, Deserialize, Serialize}; use super::Index; @@ -23,14 +23,14 @@ pub struct Settings { #[serde( default, deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", + skip_serializing_if = "Option::is_none" )] pub displayed_attributes: Option>>, #[serde( default, deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", + skip_serializing_if = "Option::is_none" )] pub searchable_attributes: Option>>, @@ -40,7 +40,7 @@ pub struct Settings { #[serde( default, deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", + skip_serializing_if = "Option::is_none" )] pub ranking_rules: Option>>, } @@ -65,8 +65,9 @@ pub struct Facets { } fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> -where T: Deserialize<'de>, - D: Deserializer<'de> +where + T: Deserialize<'de>, + D: Deserializer<'de>, { Deserialize::deserialize(deserializer).map(Some) } @@ -85,7 +86,7 @@ impl Index { let mut wtxn = self.write_txn()?; // Set the primary key if not set already, ignore if already set. - if let (None, Some(ref primary_key)) = (self.primary_key(&wtxn)?, primary_key) { + if let (None, Some(ref primary_key)) = (self.primary_key(&wtxn)?, primary_key) { self.put_primary_key(&mut wtxn, primary_key)?; } @@ -106,10 +107,11 @@ impl Index { info!("document addition done: {:?}", result); - result.and_then(|addition_result| wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into)) + result.and_then(|addition_result| { + wtxn.commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into) + }) } pub fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { @@ -210,14 +212,16 @@ impl Index { let mut builder = update_builder.delete_documents(&mut txn, self)?; // We ignore unexisting document ids - ids.iter().for_each(|id| { builder.delete_external_id(id); }); + ids.iter().for_each(|id| { + builder.delete_external_id(id); + }); match builder.execute() { Ok(deleted) => txn .commit() .and(Ok(UpdateResult::DocumentDeletion { deleted })) .map_err(Into::into), - Err(e) => Err(e) + Err(e) => Err(e), } } } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 605dee22b..d9ec8fcda 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -12,13 +12,13 @@ use heed::EnvOpenOptions; use log::debug; use serde::{Deserialize, Serialize}; use thiserror::Error; +use tokio::fs::remove_dir_all; use tokio::sync::{mpsc, oneshot, RwLock}; use tokio::task::spawn_blocking; -use tokio::fs::remove_dir_all; use uuid::Uuid; -use super::{IndexSettings, get_arc_ownership_blocking}; use super::update_handler::UpdateHandler; +use super::{get_arc_ownership_blocking, IndexSettings}; use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; use crate::index_controller::{ @@ -49,7 +49,11 @@ impl IndexMeta { let created_at = index.created_at(&txn)?; let updated_at = index.updated_at(&txn)?; let primary_key = index.primary_key(&txn)?.map(String::from); - Ok(Self { primary_key, updated_at, created_at }) + Ok(Self { + primary_key, + updated_at, + created_at, + }) } } @@ -98,7 +102,7 @@ enum IndexMsg { uuid: Uuid, index_settings: IndexSettings, ret: oneshot::Sender>, - } + }, } struct IndexActor { @@ -136,8 +140,7 @@ impl IndexActor { store: S, ) -> Result { let options = IndexerOpts::default(); - let update_handler = - UpdateHandler::new(&options).map_err(IndexError::Error)?; + let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; let update_handler = Arc::new(update_handler); let read_receiver = Some(read_receiver); let write_receiver = Some(write_receiver); @@ -241,7 +244,11 @@ impl IndexActor { GetMeta { uuid, ret } => { let _ = ret.send(self.handle_get_meta(uuid).await); } - UpdateIndex { uuid, index_settings, ret } => { + UpdateIndex { + uuid, + index_settings, + ret, + } => { let _ = ret.send(self.handle_update_index(uuid, index_settings).await); } } @@ -366,30 +373,34 @@ impl IndexActor { } } - async fn handle_update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { - let index = self.store + async fn handle_update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> Result { + let index = self + .store .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || { - match index_settings.primary_key { - Some(ref primary_key) => { - let mut txn = index.write_txn()?; - if index.primary_key(&txn)?.is_some() { - return Err(IndexError::ExistingPrimaryKey) - } - index.put_primary_key(&mut txn, primary_key)?; - let meta = IndexMeta::new_txn(&index, &txn)?; - txn.commit()?; - Ok(meta) - }, - None => { - let meta = IndexMeta::new(&index)?; - Ok(meta) - }, + spawn_blocking(move || match index_settings.primary_key { + Some(ref primary_key) => { + let mut txn = index.write_txn()?; + if index.primary_key(&txn)?.is_some() { + return Err(IndexError::ExistingPrimaryKey); + } + index.put_primary_key(&mut txn, primary_key)?; + let meta = IndexMeta::new_txn(&index, &txn)?; + txn.commit()?; + Ok(meta) } - }).await + None => { + let meta = IndexMeta::new(&index)?; + Ok(meta) + } + }) + .await .map_err(|e| IndexError::Error(e.into()))? } } @@ -416,7 +427,8 @@ impl IndexActorHandle { pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::CreateIndex { ret, + let msg = IndexMsg::CreateIndex { + ret, uuid, primary_key, }; @@ -502,10 +514,14 @@ impl IndexActorHandle { pub async fn update_index( &self, uuid: Uuid, - index_settings: IndexSettings + index_settings: IndexSettings, ) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::UpdateIndex { uuid, index_settings, ret }; + let msg = IndexMsg::UpdateIndex { + uuid, + index_settings, + ret, + }; let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -571,10 +587,7 @@ impl IndexStore for HeedIndexStore { let index = spawn_blocking(move || open_index(path, index_size)) .await .map_err(|e| IndexError::Error(e.into()))??; - self.index_store - .write() - .await - .insert(uuid, index.clone()); + self.index_store.write().await.insert(uuid, index.clone()); Ok(Some(index)) } } @@ -582,7 +595,8 @@ impl IndexStore for HeedIndexStore { async fn delete(&self, uuid: Uuid) -> Result> { let db_path = self.path.join(format!("index-{}", uuid)); - remove_dir_all(db_path).await + remove_dir_all(db_path) + .await .map_err(|e| IndexError::Error(e.into()))?; let index = self.index_store.write().await.remove(&uuid); Ok(index) @@ -590,11 +604,9 @@ impl IndexStore for HeedIndexStore { } fn open_index(path: impl AsRef, size: usize) -> Result { - create_dir_all(&path) - .map_err(|e| IndexError::Error(e.into()))?; + create_dir_all(&path).map_err(|e| IndexError::Error(e.into()))?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options, &path) - .map_err(IndexError::Error)?; + let index = milli::Index::new(options, &path).map_err(IndexError::Error)?; Ok(Index(Arc::new(index))) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index ef8b16340..0db60a41b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -9,17 +9,17 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; -use anyhow::bail; use actix_web::web::{Bytes, Payload}; +use anyhow::bail; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::time::sleep; -pub use updates::{Processed, Processing, Failed}; -use crate::index::{SearchResult, SearchQuery, Document}; -use crate::index::{UpdateResult, Settings, Facets}; +use crate::index::{Document, SearchQuery, SearchResult}; +use crate::index::{Facets, Settings, UpdateResult}; +pub use updates::{Failed, Processed, Processing}; pub type UpdateStatus = updates::UpdateStatus; @@ -51,7 +51,6 @@ pub struct IndexSettings { pub primary_key: Option, } - pub struct IndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, @@ -59,11 +58,20 @@ pub struct IndexController { } impl IndexController { - pub fn new(path: impl AsRef, index_size: usize, update_store_size: usize) -> anyhow::Result { + pub fn new( + path: impl AsRef, + index_size: usize, + update_store_size: usize, + ) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; let index_actor = index_actor::IndexActorHandle::new(&path, index_size)?; - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path, update_store_size)?; - Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) + let update_handle = + update_actor::UpdateActorHandle::new(index_actor.clone(), &path, update_store_size)?; + Ok(Self { + uuid_resolver, + index_handle: index_actor, + update_handle, + }) } pub async fn add_documents( @@ -75,7 +83,11 @@ impl IndexController { primary_key: Option, ) -> anyhow::Result { let uuid = self.uuid_resolver.get_or_create(uid).await?; - let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; + let meta = UpdateMeta::DocumentsAddition { + method, + format, + primary_key, + }; let (sender, receiver) = mpsc::channel(10); // It is necessary to spawn a local task to senf the payload to the update handle to @@ -84,10 +96,13 @@ impl IndexController { tokio::task::spawn_local(async move { while let Some(bytes) = payload.next().await { match bytes { - Ok(bytes) => { let _ = sender.send(Ok(bytes)).await; }, + Ok(bytes) => { + let _ = sender.send(Ok(bytes)).await; + } Err(e) => { let error: Box = Box::new(e); - let _ = sender.send(Err(error)).await; }, + let _ = sender.send(Err(error)).await; + } } } }); @@ -105,7 +120,11 @@ impl IndexController { Ok(status) } - pub async fn delete_documents(&self, uid: String, document_ids: Vec) -> anyhow::Result { + pub async fn delete_documents( + &self, + uid: String, + document_ids: Vec, + ) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(uid).await?; let meta = UpdateMeta::DeleteDocuments; let (sender, receiver) = mpsc::channel(10); @@ -120,7 +139,12 @@ impl IndexController { Ok(status) } - pub async fn update_settings(&self, uid: String, settings: Settings, create: bool) -> anyhow::Result { + pub async fn update_settings( + &self, + uid: String, + settings: Settings, + create: bool, + ) -> anyhow::Result { let uuid = if create { let uuid = self.uuid_resolver.get_or_create(uid).await?; // We need to create the index upfront, since it would otherwise only be created when @@ -143,9 +167,12 @@ impl IndexController { Ok(status) } - pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { - let IndexSettings { uid: name, primary_key } = index_settings; - let uid = name.unwrap(); + pub async fn create_index( + &self, + index_settings: IndexSettings, + ) -> anyhow::Result { + let IndexSettings { uid, primary_key } = index_settings; + let uid = uid.ok_or_else(|| anyhow::anyhow!("Can't create an index without a uid."))?; let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; let _ = self.update_handle.create(uuid).await?; @@ -155,25 +182,20 @@ impl IndexController { } pub async fn delete_index(&self, uid: String) -> anyhow::Result<()> { - let uuid = self.uuid_resolver - .delete(uid) - .await?; + let uuid = self.uuid_resolver.delete(uid).await?; self.update_handle.delete(uuid).await?; self.index_handle.delete(uuid).await?; Ok(()) } pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result { - let uuid = self.uuid_resolver - .resolve(uid) - .await?; + let uuid = self.uuid_resolver.resolve(uid).await?; let result = self.update_handle.update_status(uuid, id).await?; Ok(result) } pub async fn all_update_status(&self, uid: String) -> anyhow::Result> { - let uuid = self.uuid_resolver - .resolve(uid).await?; + let uuid = self.uuid_resolver.resolve(uid).await?; let result = self.update_handle.get_all_updates_status(uuid).await?; Ok(result) } @@ -193,9 +215,7 @@ impl IndexController { } pub async fn settings(&self, uid: String) -> anyhow::Result { - let uuid = self.uuid_resolver - .resolve(uid.clone()) - .await?; + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) } @@ -207,10 +227,11 @@ impl IndexController { limit: usize, attributes_to_retrieve: Option>, ) -> anyhow::Result> { - let uuid = self.uuid_resolver - .resolve(uid.clone()) + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let documents = self + .index_handle + .documents(uuid, offset, limit, attributes_to_retrieve) .await?; - let documents = self.index_handle.documents(uuid, offset, limit, attributes_to_retrieve).await?; Ok(documents) } @@ -220,21 +241,24 @@ impl IndexController { doc_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result { - let uuid = self.uuid_resolver - .resolve(uid.clone()) + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let document = self + .index_handle + .document(uuid, doc_id, attributes_to_retrieve) .await?; - let document = self.index_handle.document(uuid, doc_id, attributes_to_retrieve).await?; Ok(document) } - pub async fn update_index(&self, uid: String, index_settings: IndexSettings) -> anyhow::Result { + pub async fn update_index( + &self, + uid: String, + index_settings: IndexSettings, + ) -> anyhow::Result { if index_settings.uid.is_some() { bail!("Can't change the index uid.") } - let uuid = self.uuid_resolver - .resolve(uid.clone()) - .await?; + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let meta = self.index_handle.update_index(uuid, index_settings).await?; let meta = IndexMetadata { uid, meta }; Ok(meta) @@ -248,9 +272,7 @@ impl IndexController { pub async fn get_index(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(uid.clone()).await?; - let meta = self.index_handle - .get_index_meta(uuid) - .await?; + let meta = self.index_handle.get_index_meta(uuid).await?; let meta = IndexMetadata { uid, meta }; Ok(meta) } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 2b9d05a4b..1e788ff34 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -52,7 +52,7 @@ enum UpdateMsg { Create { uuid: Uuid, ret: oneshot::Sender>, - } + }, } struct UpdateActor { @@ -213,7 +213,11 @@ impl UpdateActorHandle where D: AsRef<[u8]> + Sized + 'static + Sync + Send, { - pub fn new(index_handle: IndexActorHandle, path: impl AsRef, update_store_size: usize) -> anyhow::Result { + pub fn new( + index_handle: IndexActorHandle, + path: impl AsRef, + update_store_size: usize, + ) -> anyhow::Result { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); let store = MapUpdateStoreStore::new(index_handle, &path, update_store_size); @@ -278,7 +282,11 @@ struct MapUpdateStoreStore { } impl MapUpdateStoreStore { - fn new(index_handle: IndexActorHandle, path: impl AsRef, update_store_size: usize) -> Self { + fn new( + index_handle: IndexActorHandle, + path: impl AsRef, + update_store_size: usize, + ) -> Self { let db = Arc::new(RwLock::new(HashMap::new())); let path = path.as_ref().to_owned(); Self { diff --git a/meilisearch-http/src/index_controller/update_handler.rs b/meilisearch-http/src/index_controller/update_handler.rs index 766dfc5f0..17f7107a2 100644 --- a/meilisearch-http/src/index_controller/update_handler.rs +++ b/meilisearch-http/src/index_controller/update_handler.rs @@ -1,14 +1,14 @@ use std::fs::File; +use crate::index::Index; use anyhow::Result; use grenad::CompressionType; use milli::update::UpdateBuilder; -use crate::index::Index; use rayon::ThreadPool; +use crate::index::UpdateResult; use crate::index_controller::updates::{Failed, Processed, Processing}; use crate::index_controller::UpdateMeta; -use crate::index::UpdateResult; use crate::option::IndexerOpts; pub struct UpdateHandler { @@ -23,9 +23,7 @@ pub struct UpdateHandler { } impl UpdateHandler { - pub fn new( - opt: &IndexerOpts, - ) -> anyhow::Result { + pub fn new(opt: &IndexerOpts) -> anyhow::Result { let thread_pool = rayon::ThreadPoolBuilder::new() .num_threads(opt.indexing_jobs.unwrap_or(0)) .build()?; @@ -59,7 +57,6 @@ impl UpdateHandler { update_builder } - pub fn handle_update( &self, meta: Processing, diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index 9ce2f242b..6de30ab7f 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -4,11 +4,11 @@ use std::sync::Arc; use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; use heed::{Database, Env, EnvOpenOptions}; +use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use std::fs::File; use tokio::sync::mpsc; use uuid::Uuid; -use parking_lot::RwLock; use crate::index_controller::updates::*; @@ -252,24 +252,27 @@ where updates.extend(pending); - let aborted = - self.aborted_meta.iter(&rtxn)? + let aborted = self + .aborted_meta + .iter(&rtxn)? .filter_map(Result::ok) .map(|(_, p)| p) .map(UpdateStatus::from); updates.extend(aborted); - let processed = - self.processed_meta.iter(&rtxn)? + let processed = self + .processed_meta + .iter(&rtxn)? .filter_map(Result::ok) .map(|(_, p)| p) .map(UpdateStatus::from); updates.extend(processed); - let failed = - self.failed_meta.iter(&rtxn)? + let failed = self + .failed_meta + .iter(&rtxn)? .filter_map(Result::ok) .map(|(_, p)| p) .map(UpdateStatus::from); @@ -372,90 +375,90 @@ where //#[cfg(test)] //mod tests { - //use super::*; - //use std::thread; - //use std::time::{Duration, Instant}; +//use super::*; +//use std::thread; +//use std::time::{Duration, Instant}; - //#[test] - //fn simple() { - //let dir = tempfile::tempdir().unwrap(); - //let mut options = EnvOpenOptions::new(); - //options.map_size(4096 * 100); - //let update_store = UpdateStore::open( - //options, - //dir, - //|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - //let new_meta = meta.meta().to_string() + " processed"; - //let processed = meta.process(new_meta); - //Ok(processed) - //}, - //) - //.unwrap(); +//#[test] +//fn simple() { +//let dir = tempfile::tempdir().unwrap(); +//let mut options = EnvOpenOptions::new(); +//options.map_size(4096 * 100); +//let update_store = UpdateStore::open( +//options, +//dir, +//|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { +//let new_meta = meta.meta().to_string() + " processed"; +//let processed = meta.process(new_meta); +//Ok(processed) +//}, +//) +//.unwrap(); - //let meta = String::from("kiki"); - //let update = update_store.register_update(meta, &[]).unwrap(); - //thread::sleep(Duration::from_millis(100)); - //let meta = update_store.meta(update.id()).unwrap().unwrap(); - //if let UpdateStatus::Processed(Processed { success, .. }) = meta { - //assert_eq!(success, "kiki processed"); - //} else { - //panic!() - //} - //} - - //#[test] - //#[ignore] - //fn long_running_update() { - //let dir = tempfile::tempdir().unwrap(); - //let mut options = EnvOpenOptions::new(); - //options.map_size(4096 * 100); - //let update_store = UpdateStore::open( - //options, - //dir, - //|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - //thread::sleep(Duration::from_millis(400)); - //let new_meta = meta.meta().to_string() + "processed"; - //let processed = meta.process(new_meta); - //Ok(processed) - //}, - //) - //.unwrap(); - - //let before_register = Instant::now(); - - //let meta = String::from("kiki"); - //let update_kiki = update_store.register_update(meta, &[]).unwrap(); - //assert!(before_register.elapsed() < Duration::from_millis(200)); - - //let meta = String::from("coco"); - //let update_coco = update_store.register_update(meta, &[]).unwrap(); - //assert!(before_register.elapsed() < Duration::from_millis(200)); - - //let meta = String::from("cucu"); - //let update_cucu = update_store.register_update(meta, &[]).unwrap(); - //assert!(before_register.elapsed() < Duration::from_millis(200)); - - //thread::sleep(Duration::from_millis(400 * 3 + 100)); - - //let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - //if let UpdateStatus::Processed(Processed { success, .. }) = meta { - //assert_eq!(success, "kiki processed"); - //} else { - //panic!() - //} - - //let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - //if let UpdateStatus::Processed(Processed { success, .. }) = meta { - //assert_eq!(success, "coco processed"); - //} else { - //panic!() - //} - - //let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - //if let UpdateStatus::Processed(Processed { success, .. }) = meta { - //assert_eq!(success, "cucu processed"); - //} else { - //panic!() - //} - //} +//let meta = String::from("kiki"); +//let update = update_store.register_update(meta, &[]).unwrap(); +//thread::sleep(Duration::from_millis(100)); +//let meta = update_store.meta(update.id()).unwrap().unwrap(); +//if let UpdateStatus::Processed(Processed { success, .. }) = meta { +//assert_eq!(success, "kiki processed"); +//} else { +//panic!() +//} +//} + +//#[test] +//#[ignore] +//fn long_running_update() { +//let dir = tempfile::tempdir().unwrap(); +//let mut options = EnvOpenOptions::new(); +//options.map_size(4096 * 100); +//let update_store = UpdateStore::open( +//options, +//dir, +//|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { +//thread::sleep(Duration::from_millis(400)); +//let new_meta = meta.meta().to_string() + "processed"; +//let processed = meta.process(new_meta); +//Ok(processed) +//}, +//) +//.unwrap(); + +//let before_register = Instant::now(); + +//let meta = String::from("kiki"); +//let update_kiki = update_store.register_update(meta, &[]).unwrap(); +//assert!(before_register.elapsed() < Duration::from_millis(200)); + +//let meta = String::from("coco"); +//let update_coco = update_store.register_update(meta, &[]).unwrap(); +//assert!(before_register.elapsed() < Duration::from_millis(200)); + +//let meta = String::from("cucu"); +//let update_cucu = update_store.register_update(meta, &[]).unwrap(); +//assert!(before_register.elapsed() < Duration::from_millis(200)); + +//thread::sleep(Duration::from_millis(400 * 3 + 100)); + +//let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); +//if let UpdateStatus::Processed(Processed { success, .. }) = meta { +//assert_eq!(success, "kiki processed"); +//} else { +//panic!() +//} + +//let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); +//if let UpdateStatus::Processed(Processed { success, .. }) = meta { +//assert_eq!(success, "coco processed"); +//} else { +//panic!() +//} + +//let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); +//if let UpdateStatus::Processed(Processed { success, .. }) = meta { +//assert_eq!(success, "cucu processed"); +//} else { +//panic!() +//} +//} //} diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index b2ad54a14..3c70aee71 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -1,5 +1,5 @@ -use chrono::{Utc, DateTime}; -use serde::{Serialize, Deserialize}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 69e9a626b..2ee9c6b17 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -1,6 +1,9 @@ use std::{fs::create_dir_all, path::Path}; -use heed::{Database, Env, EnvOpenOptions, types::{ByteSlice, Str}}; +use heed::{ + types::{ByteSlice, Str}, + Database, Env, EnvOpenOptions, +}; use log::{info, warn}; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; @@ -73,14 +76,14 @@ impl UuidResolverActor { async fn handle_create(&self, uid: String) -> Result { if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)) + return Err(UuidError::BadlyFormatted(uid)); } self.store.create_uuid(uid, true).await } async fn handle_get_or_create(&self, uid: String) -> Result { if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)) + return Err(UuidError::BadlyFormatted(uid)); } self.store.create_uuid(uid, false).await } @@ -106,7 +109,8 @@ impl UuidResolverActor { } fn is_index_uid_valid(uid: &str) -> bool { - uid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') + uid.chars() + .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') } #[derive(Clone)] @@ -235,7 +239,8 @@ impl UuidStore for HeedUuidStore { Ok(uuid) } } - }).await? + }) + .await? } async fn get_uuid(&self, name: String) -> Result> { @@ -250,7 +255,8 @@ impl UuidStore for HeedUuidStore { } None => Ok(None), } - }).await? + }) + .await? } async fn delete(&self, uid: String) -> Result> { @@ -265,9 +271,10 @@ impl UuidStore for HeedUuidStore { txn.commit()?; Ok(Some(uuid)) } - None => Ok(None) + None => Ok(None), } - }).await? + }) + .await? } async fn list(&self) -> Result> { @@ -282,6 +289,7 @@ impl UuidStore for HeedUuidStore { entries.push((name.to_owned(), uuid)) } Ok(entries) - }).await? + }) + .await? } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index da1aee746..cdd874a35 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,63 +1,59 @@ pub mod data; pub mod error; pub mod helpers; -pub mod option; -pub mod routes; mod index; mod index_controller; +pub mod option; +pub mod routes; -pub use option::Opt; pub use self::data::Data; +pub use option::Opt; #[macro_export] macro_rules! create_app { - ($data:expr, $enable_frontend:expr) => { - { - use actix_cors::Cors; - use actix_web::App; - use actix_web::middleware::TrailingSlash; - use actix_web::{web, middleware}; - use meilisearch_http::error::payload_error_handler; - use meilisearch_http::routes::*; + ($data:expr, $enable_frontend:expr) => {{ + use actix_cors::Cors; + use actix_web::middleware::TrailingSlash; + use actix_web::App; + use actix_web::{middleware, web}; + use meilisearch_http::error::payload_error_handler; + use meilisearch_http::routes::*; - let app = App::new() - .data($data.clone()) - .app_data( - web::JsonConfig::default() + let app = App::new() + .data($data.clone()) + .app_data( + web::JsonConfig::default() .limit($data.http_payload_size_limit()) .content_type(|_mime| true) // Accept all mime types .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .app_data( - web::QueryConfig::default() - .error_handler(|err, _req| payload_error_handler(err).into()) - ) - .configure(document::services) - .configure(index::services) - .configure(search::services) - .configure(settings::services) - .configure(stop_words::services) - .configure(synonym::services) - .configure(health::services) - .configure(stats::services) - .configure(key::services); - //.configure(routes::dump::services); - let app = if $enable_frontend { - app - .service(load_html) - .service(load_css) - } else { - app - }; - app.wrap( - Cors::default() + ) + .app_data( + web::QueryConfig::default() + .error_handler(|err, _req| payload_error_handler(err).into()), + ) + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(stop_words::services) + .configure(synonym::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services); + //.configure(routes::dump::services); + let app = if $enable_frontend { + app.service(load_html).service(load_css) + } else { + app + }; + app.wrap( + Cors::default() .send_wildcard() .allowed_headers(vec!["content-type", "x-meili-api-key"]) - .max_age(86_400) // 24h - ) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - } - }; + .max_age(86_400), // 24h + ) + .wrap(middleware::Logger::default()) + .wrap(middleware::Compress::default()) + .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) + }}; } diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 36959c5b3..9415f52d5 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -2,7 +2,7 @@ use std::env; use actix_web::HttpServer; use main_error::MainError; -use meilisearch_http::{Data, Opt, create_app}; +use meilisearch_http::{create_app, Data, Opt}; use structopt::StructOpt; //mod analytics; @@ -44,29 +44,30 @@ async fn main() -> Result<(), MainError> { } } "development" => { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .init(); } _ => unreachable!(), } //if let Some(path) = &opt.import_snapshot { - //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; + //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; //} let data = Data::new(opt.clone())?; //if !opt.no_analytics { - //let analytics_data = data.clone(); - //let analytics_opt = opt.clone(); - //thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); + //let analytics_data = data.clone(); + //let analytics_opt = opt.clone(); + //thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); //} //if let Some(path) = &opt.import_dump { - //dump::import_dump(&data, path, opt.dump_batch_size)?; + //dump::import_dump(&data, path, opt.dump_batch_size)?; //} //if opt.schedule_snapshot { - //snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; + //snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; //} print_launch_resume(&opt, &data); @@ -78,11 +79,14 @@ async fn main() -> Result<(), MainError> { Ok(()) } -async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box> { - +async fn run_http( + data: Data, + opt: Opt, + enable_frontend: bool, +) -> Result<(), Box> { let http_server = HttpServer::new(move || create_app!(&data, enable_frontend)) - // Disable signals allows the server to terminate immediately when a user enter CTRL-C - .disable_signals(); + // Disable signals allows the server to terminate immediately when a user enter CTRL-C + .disable_signals(); if let Some(config) = opt.get_ssl_config()? { http_server @@ -95,7 +99,6 @@ async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box Ok(()) } - pub fn print_launch_resume(opt: &Opt, data: &Data) { let ascii_name = r#" 888b d888 d8b 888 d8b .d8888b. 888 diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 0a1d494de..82eb75fc1 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -1,15 +1,15 @@ -use std::{error, fs}; use std::io::{BufReader, Read}; use std::path::PathBuf; use std::sync::Arc; +use std::{error, fs}; use byte_unit::Byte; +use grenad::CompressionType; use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; use rustls::{ AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, RootCertStore, }; -use grenad::CompressionType; use structopt::StructOpt; #[derive(Debug, Clone, StructOpt)] @@ -99,7 +99,11 @@ pub struct Opt { /// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project. /// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable. #[cfg(all(not(debug_assertions), feature = "sentry"))] - #[structopt(long, env = "SENTRY_DSN", default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337")] + #[structopt( + long, + env = "SENTRY_DSN", + default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337" + )] pub sentry_dsn: String, /// Disable Sentry error reporting. diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index c53c3cb85..59bc5fe0e 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -7,10 +7,10 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; use serde_json::Value; -use crate::Data; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; +use crate::Data; const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; @@ -19,7 +19,10 @@ macro_rules! guard_content_type { ($fn_name:ident, $guard_value:literal) => { fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { if let Some(content_type) = head.headers.get("Content-Type") { - content_type.to_str().map(|v| v.contains($guard_value)).unwrap_or(false) + content_type + .to_str() + .map(|v| v.contains($guard_value)) + .unwrap_or(false) } else { false } @@ -57,7 +60,10 @@ async fn get_document( ) -> Result { let index = path.index_uid.clone(); let id = path.document_id.clone(); - match data.retrieve_document(index, id, None as Option>).await { + match data + .retrieve_document(index, id, None as Option>) + .await + { Ok(document) => { let json = serde_json::to_string(&document).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -76,7 +82,10 @@ async fn delete_document( data: web::Data, path: web::Path, ) -> Result { - match data.delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]).await { + match data + .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]) + .await + { Ok(result) => { let json = serde_json::to_string(&result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -104,16 +113,17 @@ async fn get_all_documents( let attributes_to_retrieve = params .attributes_to_retrieve .as_ref() - .map(|attrs| attrs - .split(',') - .map(String::from) - .collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); - match data.retrieve_documents( - path.index_uid.clone(), - params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), - params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), - attributes_to_retrieve).await { + match data + .retrieve_documents( + path.index_uid.clone(), + params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), + params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), + attributes_to_retrieve, + ) + .await + { Ok(docs) => { let json = serde_json::to_string(&docs).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -149,7 +159,8 @@ async fn add_documents_json( UpdateFormat::Json, body, params.primary_key.clone(), - ).await; + ) + .await; match addition_result { Ok(update) => { @@ -163,7 +174,6 @@ async fn add_documents_json( } } - /// Default route for adding documents, this should return an error and redirect to the documentation #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn add_documents_default( @@ -191,7 +201,7 @@ async fn update_documents_default( #[put( "/indexes/{index_uid}/documents", wrap = "Authentication::Private", - guard = "guard_json", + guard = "guard_json" )] async fn update_documents( data: web::Data, @@ -206,7 +216,8 @@ async fn update_documents( UpdateFormat::Json, body, params.primary_key.clone(), - ).await; + ) + .await; match addition_result { Ok(update) => { @@ -231,7 +242,11 @@ async fn delete_documents( ) -> Result { let ids = body .iter() - .map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) + .map(|v| { + v.as_str() + .map(String::from) + .unwrap_or_else(|| v.to_string()) + }) .collect(); match data.delete_documents(path.index_uid.clone(), ids).await { diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index cb1c08e24..c7c9e521a 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -3,10 +3,10 @@ use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::Data; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; +use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(list_indexes) @@ -18,7 +18,6 @@ pub fn services(cfg: &mut web::ServiceConfig) { .service(get_all_updates_status); } - #[get("/indexes", wrap = "Authentication::Private")] async fn list_indexes(data: web::Data) -> Result { match data.list_indexes().await { @@ -96,7 +95,10 @@ async fn update_index( body: web::Json, ) -> Result { let body = body.into_inner(); - match data.update_index(path.into_inner().index_uid, body.primary_key, body.uid).await { + match data + .update_index(path.into_inner().index_uid, body.primary_key, body.uid) + .await + { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -135,7 +137,9 @@ async fn get_update_status( path: web::Path, ) -> Result { let params = path.into_inner(); - let result = data.get_update_status(params.index_uid, params.update_id).await; + let result = data + .get_update_status(params.index_uid, params.update_id) + .await; match result { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); diff --git a/meilisearch-http/src/routes/key.rs b/meilisearch-http/src/routes/key.rs index a051ee0f9..b44d747c8 100644 --- a/meilisearch-http/src/routes/key.rs +++ b/meilisearch-http/src/routes/key.rs @@ -1,6 +1,6 @@ +use actix_web::get; use actix_web::web; use actix_web::HttpResponse; -use actix_web::get; use serde::Serialize; use crate::helpers::Authentication; diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 2e718ebe9..c98a5ac51 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -6,9 +6,9 @@ use serde::Deserialize; use crate::error::ResponseError; use crate::helpers::Authentication; +use crate::index::{SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; -use crate::index::{SearchQuery, DEFAULT_SEARCH_LIMIT}; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(search_with_post).service(search_with_url_query); @@ -80,7 +80,9 @@ async fn search_with_url_query( let query: SearchQuery = match params.into_inner().try_into() { Ok(q) => q, Err(e) => { - return Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + return Ok( + HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() })) + ) } }; let search_result = data.search(path.into_inner().index_uid, query).await; @@ -101,7 +103,9 @@ async fn search_with_post( path: web::Path, params: web::Json, ) -> Result { - let search_result = data.search(path.into_inner().index_uid, params.into_inner()).await; + let search_result = data + .search(path.into_inner().index_uid, params.into_inner()) + .await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index df2fe9f15..5a6246f8c 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -1,9 +1,9 @@ -use actix_web::{web, HttpResponse, delete, get, post}; +use actix_web::{delete, get, post, web, HttpResponse}; -use crate::Data; use crate::error::ResponseError; -use crate::index::Settings; use crate::helpers::Authentication; +use crate::index::Settings; +use crate::Data; #[macro_export] macro_rules! make_setting_route { @@ -98,15 +98,15 @@ make_setting_route!( ); //make_setting_route!( - //"/indexes/{index_uid}/settings/distinct-attribute", - //String, - //distinct_attribute +//"/indexes/{index_uid}/settings/distinct-attribute", +//String, +//distinct_attribute //); //make_setting_route!( - //"/indexes/{index_uid}/settings/ranking-rules", - //Vec, - //ranking_rules +//"/indexes/{index_uid}/settings/ranking-rules", +//Vec, +//ranking_rules //); macro_rules! create_services { @@ -137,7 +137,10 @@ async fn update_all( index_uid: web::Path, body: web::Json, ) -> Result { - match data.update_settings(index_uid.into_inner(), body.into_inner(), true).await { + match data + .update_settings(index_uid.into_inner(), body.into_inner(), true) + .await + { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -170,7 +173,10 @@ async fn delete_all( index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); - match data.update_settings(index_uid.into_inner(), settings, false).await { + match data + .update_settings(index_uid.into_inner(), settings, false) + .await + { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -180,4 +186,3 @@ async fn delete_all( } } } - diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index cba64194b..eee6536c6 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,8 +1,8 @@ -use std::collections::{HashMap, BTreeMap}; +use std::collections::{BTreeMap, HashMap}; +use actix_web::get; use actix_web::web; use actix_web::HttpResponse; -use actix_web::get; use chrono::{DateTime, Utc}; use serde::Serialize; diff --git a/meilisearch-http/src/routes/stop_words.rs b/meilisearch-http/src/routes/stop_words.rs index 5b116e829..8f89b6425 100644 --- a/meilisearch-http/src/routes/stop_words.rs +++ b/meilisearch-http/src/routes/stop_words.rs @@ -1,5 +1,5 @@ -use actix_web::{web, HttpResponse}; use actix_web::{delete, get, post}; +use actix_web::{web, HttpResponse}; use std::collections::BTreeSet; use crate::error::ResponseError; diff --git a/meilisearch-http/src/routes/synonym.rs b/meilisearch-http/src/routes/synonym.rs index 43a0e4aeb..4106e3754 100644 --- a/meilisearch-http/src/routes/synonym.rs +++ b/meilisearch-http/src/routes/synonym.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; -use actix_web::{web, HttpResponse}; use actix_web::{delete, get, post}; +use actix_web::{web, HttpResponse}; use crate::error::ResponseError; use crate::helpers::Authentication; diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index a14769ee1..8fda99ef9 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -19,7 +19,10 @@ impl Index<'_> { pub async fn load_test_set(&self) -> u64 { let url = format!("/indexes/{}/documents", self.uid); - let (response, code) = self.service.post_str(url, include_str!("../assets/test_set.json")).await; + let (response, code) = self + .service + .post_str(url, include_str!("../assets/test_set.json")) + .await; assert_eq!(code, 200); let update_id = response["updateId"].as_i64().unwrap(); self.wait_update_id(update_id as u64).await; @@ -60,7 +63,11 @@ impl Index<'_> { self.service.post(url, documents).await } - pub async fn update_documents(&self, documents: Value, primary_key: Option<&str>) -> (Value, StatusCode) { + pub async fn update_documents( + &self, + documents: Value, + primary_key: Option<&str>, + ) -> (Value, StatusCode) { let url = match primary_key { Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key), None => format!("/indexes/{}/documents", self.uid), @@ -95,7 +102,11 @@ impl Index<'_> { self.service.get(url).await } - pub async fn get_document(&self, id: u64, _options: Option) -> (Value, StatusCode) { + pub async fn get_document( + &self, + id: u64, + _options: Option, + ) -> (Value, StatusCode) { let url = format!("/indexes/{}/documents/{}", self.uid, id); self.service.get(url).await } @@ -111,7 +122,10 @@ impl Index<'_> { } if let Some(attributes_to_retrieve) = options.attributes_to_retrieve { - url.push_str(&format!("attributesToRetrieve={}&", attributes_to_retrieve.join(","))); + url.push_str(&format!( + "attributesToRetrieve={}&", + attributes_to_retrieve.join(",") + )); } self.service.get(url).await @@ -129,7 +143,9 @@ impl Index<'_> { pub async fn delete_batch(&self, ids: Vec) -> (Value, StatusCode) { let url = format!("/indexes/{}/documents/delete-batch", self.uid); - self.service.post(url, serde_json::to_value(&ids).unwrap()).await + self.service + .post(url, serde_json::to_value(&ids).unwrap()) + .await } pub async fn settings(&self) -> (Value, StatusCode) { diff --git a/meilisearch-http/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs index fd461e61c..d1874ae84 100644 --- a/meilisearch-http/tests/common/mod.rs +++ b/meilisearch-http/tests/common/mod.rs @@ -2,8 +2,8 @@ mod index; mod server; mod service; +pub use index::{GetAllDocumentsOptions, GetDocumentOptions}; pub use server::Server; -pub use index::{GetDocumentOptions, GetAllDocumentsOptions}; /// Performs a search test on both post and get routes #[macro_export] diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 090524601..6c7225a8f 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -5,7 +5,7 @@ use tempdir::TempDir; use urlencoding::encode; use meilisearch_http::data::Data; -use meilisearch_http::option::{Opt, IndexerOpts}; +use meilisearch_http::option::{IndexerOpts, Opt}; use super::index::Index; use super::service::Service; @@ -55,10 +55,7 @@ impl Server { let data = Data::new(opt).unwrap(); let service = Service(data); - Server { - service, - _dir: dir, - } + Server { service, _dir: dir } } /// Returns a view to an index. There is no guarantee that the index exists. diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index feff49cad..b9bbffc05 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -1,15 +1,14 @@ use actix_web::{http::StatusCode, test}; use serde_json::Value; -use meilisearch_http::data::Data; use meilisearch_http::create_app; +use meilisearch_http::data::Data; pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -24,9 +23,12 @@ impl Service { } /// Send a test post request from a text body, with a `content-type:application/json` header. - pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + pub async fn post_str( + &self, + url: impl AsRef, + body: impl AsRef, + ) -> (Value, StatusCode) { + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -42,8 +44,7 @@ impl Service { } pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; @@ -55,8 +56,7 @@ impl Service { } pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::put() .uri(url.as_ref()) @@ -71,8 +71,7 @@ impl Service { } pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 37b06f46f..dd10541b5 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -1,7 +1,7 @@ -use serde_json::{json, Value}; use chrono::DateTime; +use serde_json::{json, Value}; -use crate::common::{Server, GetAllDocumentsOptions}; +use crate::common::{GetAllDocumentsOptions, Server}; #[actix_rt::test] async fn add_documents_no_index_creation() { @@ -32,9 +32,12 @@ async fn add_documents_no_index_creation() { assert_eq!(response["updateId"], 0); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); - let processed_at = DateTime::parse_from_rfc3339(response["processedAt"].as_str().unwrap()).unwrap(); - let enqueued_at = DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap(); - let started_processing_at = DateTime::parse_from_rfc3339(response["startedProcessingAt"].as_str().unwrap()).unwrap(); + let processed_at = + DateTime::parse_from_rfc3339(response["processedAt"].as_str().unwrap()).unwrap(); + let enqueued_at = + DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap(); + let started_processing_at = + DateTime::parse_from_rfc3339(response["startedProcessingAt"].as_str().unwrap()).unwrap(); assert!(processed_at > started_processing_at); assert!(started_processing_at > enqueued_at); @@ -71,7 +74,8 @@ async fn document_addition_with_primary_key() { "content": "foo", } ]); - let (_response, code) = index.add_documents(documents, Some("primary")).await; assert_eq!(code, 200); + let (_response, code) = index.add_documents(documents, Some("primary")).await; + assert_eq!(code, 200); index.wait_update_id(0).await; @@ -97,7 +101,8 @@ async fn document_update_with_primary_key() { "content": "foo", } ]); - let (_response, code) = index.update_documents(documents, Some("primary")).await; assert_eq!(code, 200); + let (_response, code) = index.update_documents(documents, Some("primary")).await; + assert_eq!(code, 200); index.wait_update_id(0).await; @@ -158,7 +163,7 @@ async fn update_documents_with_primary_key_and_primary_key_already_exists() { assert_eq!(code, 200); index.wait_update_id(0).await; -let (response, code) = index.get_update(0).await; + let (response, code) = index.get_update(0).await; assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["updateId"], 0); @@ -263,7 +268,10 @@ async fn update_document() { let (response, code) = index.get_document(1, None).await; assert_eq!(code, 200); - assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##); + assert_eq!( + response.to_string(), + r##"{"doc_id":1,"content":"foo","other":"bar"}"## + ); } #[actix_rt::test] @@ -275,7 +283,12 @@ async fn add_larger_dataset() { assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(1000), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(1000), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 77); } @@ -291,7 +304,12 @@ async fn update_larger_dataset() { assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(1000), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(1000), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 77); } diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs index 3dba7e6fd..e7a01acd4 100644 --- a/meilisearch-http/tests/documents/delete_documents.rs +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -1,6 +1,6 @@ use serde_json::json; -use crate::common::{Server, GetAllDocumentsOptions}; +use crate::common::{GetAllDocumentsOptions, Server}; #[actix_rt::test] async fn delete_one_document_unexisting_index() { @@ -24,7 +24,9 @@ async fn delete_one_unexisting_document() { async fn delete_one_document() { let server = Server::new().await; let index = server.index("test"); - index.add_documents(json!([{ "id": 0, "content": "foobar" }]), None).await; + index + .add_documents(json!([{ "id": 0, "content": "foobar" }]), None) + .await; index.wait_update_id(0).await; let (_response, code) = server.index("test").delete_document(0).await; assert_eq!(code, 200); @@ -39,20 +41,26 @@ async fn clear_all_documents_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").clear_all_documents().await; assert_eq!(code, 400); - } #[actix_rt::test] async fn clear_all_documents() { let server = Server::new().await; let index = server.index("test"); - index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }]), None).await; + index + .add_documents( + json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }]), + None, + ) + .await; index.wait_update_id(0).await; let (_response, code) = index.clear_all_documents().await; assert_eq!(code, 200); let _update = index.wait_update_id(1).await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } @@ -67,7 +75,9 @@ async fn clear_all_documents_empty_index() { assert_eq!(code, 200); let _update = index.wait_update_id(0).await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } @@ -89,13 +99,14 @@ async fn delete_batch() { assert_eq!(code, 200); let _update = index.wait_update_id(1).await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 1); assert_eq!(response.as_array().unwrap()[0]["id"], 3); } - #[actix_rt::test] async fn delete_no_document_batch() { let server = Server::new().await; @@ -106,7 +117,9 @@ async fn delete_no_document_batch() { assert_eq!(code, 200); let _update = index.wait_update_id(1).await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 3); } diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index c9ed83316..d54234860 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -1,5 +1,5 @@ -use crate::common::Server; use crate::common::GetAllDocumentsOptions; +use crate::common::Server; use serde_json::json; @@ -8,10 +8,7 @@ use serde_json::json; #[actix_rt::test] async fn get_unexisting_index_single_document() { let server = Server::new().await; - let (_response, code) = server - .index("test") - .get_document(1, None) - .await; + let (_response, code) = server.index("test").get_document(1, None).await; assert_eq!(code, 400); } @@ -20,9 +17,7 @@ async fn get_unexisting_document() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; - let (_response, code) = index - .get_document(1, None) - .await; + let (_response, code) = index.get_document(1, None).await; assert_eq!(code, 400); } @@ -40,14 +35,15 @@ async fn get_document() { let (_, code) = index.add_documents(documents, None).await; assert_eq!(code, 200); index.wait_update_id(0).await; - let (response, code) = index - .get_document(0, None) - .await; + let (response, code) = index.get_document(0, None).await; assert_eq!(code, 200); - assert_eq!(response, serde_json::json!( { - "id": 0, - "content": "foobar", - })); + assert_eq!( + response, + serde_json::json!( { + "id": 0, + "content": "foobar", + }) + ); } #[actix_rt::test] @@ -67,7 +63,9 @@ async fn get_no_documents() { let (_, code) = index.create(None).await; assert_eq!(code, 200); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } @@ -78,7 +76,9 @@ async fn get_all_documents_no_options() { let index = server.index("test"); index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); let arr = response.as_array().unwrap(); assert_eq!(arr.len(), 20); @@ -109,7 +109,12 @@ async fn test_get_all_documents_limit() { let index = server.index("test"); index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(5), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(5), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 5); assert_eq!(response.as_array().unwrap()[0]["id"], 0); @@ -121,7 +126,12 @@ async fn test_get_all_documents_offset() { let index = server.index("test"); index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { offset: Some(5), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + offset: Some(5), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); assert_eq!(response.as_array().unwrap()[0]["id"], 13); @@ -133,35 +143,90 @@ async fn test_get_all_documents_attributes_to_retrieve() { let index = server.index("test"); index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec!["name"]), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["name"]), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 1); - assert!(response.as_array().unwrap()[0].as_object().unwrap().get("name").is_some()); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 1 + ); + assert!(response.as_array().unwrap()[0] + .as_object() + .unwrap() + .get("name") + .is_some()); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec![]), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec![]), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 0); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 0 + ); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec!["name", "tags"]), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["name", "tags"]), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 2); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 2 + ); } #[actix_rt::test] async fn get_documents_displayed_attributes() { let server = Server::new().await; let index = server.index("test"); - index.update_settings(json!({"displayedAttributes": ["gender"]})).await; + index + .update_settings(json!({"displayedAttributes": ["gender"]})) + .await; index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 1); - assert!(response.as_array().unwrap()[0].as_object().unwrap().get("gender").is_some()); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 1 + ); + assert!(response.as_array().unwrap()[0] + .as_object() + .unwrap() + .get("gender") + .is_some()); let (response, code) = index.get_document(0, None).await; assert_eq!(code, 200); diff --git a/meilisearch-http/tests/documents/mod.rs b/meilisearch-http/tests/documents/mod.rs index ea0e39c69..a791a596f 100644 --- a/meilisearch-http/tests/documents/mod.rs +++ b/meilisearch-http/tests/documents/mod.rs @@ -1,3 +1,3 @@ mod add_documents; -mod get_documents; mod delete_documents; +mod get_documents; diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 8f7224602..003bbfc58 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -7,7 +7,6 @@ async fn create_index_no_primary_key() { let index = server.index("test"); let (response, code) = index.create(None).await; - assert_eq!(code, 200); assert_eq!(response["uid"], "test"); assert!(response.get("createdAt").is_some()); diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 9d145899e..39e79daaf 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -6,12 +6,10 @@ async fn create_and_delete_index() { let index = server.index("test"); let (_response, code) = index.create(None).await; - assert_eq!(code, 200); let (_response, code) = index.delete().await; - assert_eq!(code, 200); assert_eq!(index.get().await.1, 400); diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index 1a1815dfe..2ba7b86e4 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -52,6 +52,12 @@ async fn list_multiple_indexes() { assert!(response.is_array()); let arr = response.as_array().unwrap(); assert_eq!(arr.len(), 2); - assert!(arr.iter().find(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null).is_some()); - assert!(arr.iter().find(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key").is_some()); + assert!(arr + .iter() + .find(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null) + .is_some()); + assert!(arr + .iter() + .find(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key") + .is_some()); } diff --git a/meilisearch-http/tests/index/mod.rs b/meilisearch-http/tests/index/mod.rs index 253afcfc4..c9804c160 100644 --- a/meilisearch-http/tests/index/mod.rs +++ b/meilisearch-http/tests/index/mod.rs @@ -1,4 +1,4 @@ mod create_index; +mod delete_index; mod get_index; mod update_index; -mod delete_index; diff --git a/meilisearch-http/tests/integration.rs b/meilisearch-http/tests/integration.rs index 7e54d8bbf..7578a8dc1 100644 --- a/meilisearch-http/tests/integration.rs +++ b/meilisearch-http/tests/integration.rs @@ -1,8 +1,8 @@ mod common; +mod documents; mod index; mod search; mod settings; -mod documents; mod updates; // Tests are isolated by features in different modules to allow better readability, test diff --git a/meilisearch-http/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs index 735a3478f..56ec6439c 100644 --- a/meilisearch-http/tests/search/mod.rs +++ b/meilisearch-http/tests/search/mod.rs @@ -1,3 +1,2 @@ // This modules contains all the test concerning search. Each particular feture of the search // should be tested in its own module to isolate tests and keep the tests readable. - diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 0a8972108..e4c38cb00 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -21,7 +21,17 @@ async fn get_settings() { assert_eq!(settings["searchableAttributes"], json!(["*"])); println!("{:?}", settings); assert_eq!(settings["attributesForFaceting"], json!({})); - assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximity", "attribute", "wordsPosition", "exactness"])); + assert_eq!( + settings["rankingRules"], + json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ]) + ); } #[actix_rt::test] @@ -36,20 +46,24 @@ async fn update_settings_unknown_field() { async fn test_partial_update() { let server = Server::new().await; let index = server.index("test"); - let (_response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + let (_response, _code) = index + .update_settings(json!({"displayedAttributes": ["foo"]})) + .await; index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"],json!(["foo"])); - assert_eq!(response["searchableAttributes"],json!(["*"])); + assert_eq!(response["displayedAttributes"], json!(["foo"])); + assert_eq!(response["searchableAttributes"], json!(["*"])); - let (_response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + let (_response, _) = index + .update_settings(json!({"searchableAttributes": ["bar"]})) + .await; index.wait_update_id(1).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"],json!(["foo"])); - assert_eq!(response["searchableAttributes"],json!(["bar"])); + assert_eq!(response["displayedAttributes"], json!(["foo"])); + assert_eq!(response["searchableAttributes"], json!(["bar"])); } #[actix_rt::test] @@ -64,20 +78,22 @@ async fn delete_settings_unexisting_index() { async fn reset_all_settings() { let server = Server::new().await; let index = server.index("test"); - index.update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"]})).await; + index + .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"]})) + .await; index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"],json!(["foo"])); - assert_eq!(response["searchableAttributes"],json!(["bar"])); + assert_eq!(response["displayedAttributes"], json!(["foo"])); + assert_eq!(response["searchableAttributes"], json!(["bar"])); index.delete_settings().await; index.wait_update_id(1).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"],json!(["*"])); - assert_eq!(response["searchableAttributes"],json!(["*"])); + assert_eq!(response["displayedAttributes"], json!(["*"])); + assert_eq!(response["searchableAttributes"], json!(["*"])); } #[actix_rt::test] @@ -149,4 +165,5 @@ macro_rules! test_setting_routes { test_setting_routes!( attributes_for_faceting, displayed_attributes, - searchable_attributes); + searchable_attributes +); diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 64b5b560e..6ce5ca381 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -21,13 +21,15 @@ async fn get_update_status() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; - index.add_documents( - serde_json::json!([{ - "id": 1, - "content": "foobar", - }]), - None - ).await; + index + .add_documents( + serde_json::json!([{ + "id": 1, + "content": "foobar", + }]), + None, + ) + .await; let (_response, code) = index.get_update(0).await; assert_eq!(code, 200); // TODO check resonse format, as per #48 @@ -55,10 +57,12 @@ async fn list_updates() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; - index.add_documents( - serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), - None - ).await; + index + .add_documents( + serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), + None, + ) + .await; let (response, code) = index.list_updates().await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 1); From 94bd14ede3c85c176e1cb72e05e7913eaaa89ac2 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 18:35:16 +0100 Subject: [PATCH 181/522] add name to index_metadata --- meilisearch-http/src/index_controller/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 0db60a41b..90c67e2b6 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -27,6 +27,7 @@ pub type UpdateStatus = updates::UpdateStatus; #[serde(rename_all = "camelCase")] pub struct IndexMetadata { uid: String, + name: String, #[serde(flatten)] meta: index_actor::IndexMeta, } @@ -176,7 +177,7 @@ impl IndexController { let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; let _ = self.update_handle.create(uuid).await?; - let meta = IndexMetadata { uid, meta }; + let meta = IndexMetadata { name: uid.clone(), uid, meta }; Ok(meta) } @@ -207,7 +208,7 @@ impl IndexController { for (uid, uuid) in uuids { let meta = self.index_handle.get_index_meta(uuid).await?; - let meta = IndexMetadata { uid, meta }; + let meta = IndexMetadata { name: uid.clone(), uid, meta }; ret.push(meta); } @@ -260,7 +261,7 @@ impl IndexController { let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let meta = self.index_handle.update_index(uuid, index_settings).await?; - let meta = IndexMetadata { uid, meta }; + let meta = IndexMetadata { name: uid.clone(), uid, meta }; Ok(meta) } @@ -273,7 +274,7 @@ impl IndexController { pub async fn get_index(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let meta = self.index_handle.get_index_meta(uuid).await?; - let meta = IndexMetadata { uid, meta }; + let meta = IndexMetadata { name: uid.clone(), uid, meta }; Ok(meta) } } From 07bb1e2c4e81ddd4f4089c584e933da1680f9ad5 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 18:38:13 +0100 Subject: [PATCH 182/522] fix tests --- meilisearch-http/tests/index/create_index.rs | 6 ++++-- meilisearch-http/tests/index/get_index.rs | 3 ++- meilisearch-http/tests/index/update_index.rs | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 003bbfc58..a61524d04 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -9,11 +9,12 @@ async fn create_index_no_primary_key() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); + assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], Value::Null); - assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(response.as_object().unwrap().len(), 5); } #[actix_rt::test] @@ -24,11 +25,12 @@ async fn create_index_with_primary_key() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); + assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); //assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], "primary"); - assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(response.as_object().unwrap().len(), 5); } // TODO: partial test since we are testing error, amd error is not yet fully implemented in diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index 2ba7b86e4..8671d85cd 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -13,11 +13,12 @@ async fn create_and_get_index() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); + assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], Value::Null); - assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(response.as_object().unwrap().len(), 5); } // TODO: partial test since we are testing error, amd error is not yet fully implemented in diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index 36670682a..9c78ccb16 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -13,6 +13,7 @@ async fn update_primary_key() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); + assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); @@ -21,7 +22,7 @@ async fn update_primary_key() { assert!(created_at < updated_at); assert_eq!(response["primaryKey"], "primary"); - assert_eq!(response.as_object().unwrap().len(), 4); + assert_eq!(response.as_object().unwrap().len(), 5); } #[actix_rt::test] From 58fab035bbddfd091989ed8cfdb12d384f525e12 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 18:44:33 +0100 Subject: [PATCH 183/522] delete index returns 204 instead of 200 --- meilisearch-http/src/routes/index.rs | 2 +- meilisearch-http/tests/index/delete_index.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index c7c9e521a..29ba3d0d8 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -115,7 +115,7 @@ async fn delete_index( path: web::Path, ) -> Result { match data.delete_index(path.index_uid.clone()).await { - Ok(_) => Ok(HttpResponse::Ok().finish()), + Ok(_) => Ok(HttpResponse::NoContent().finish()), Err(e) => { Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 39e79daaf..0d1a7c820 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -10,7 +10,7 @@ async fn create_and_delete_index() { let (_response, code) = index.delete().await; - assert_eq!(code, 200); + assert_eq!(code, 204); assert_eq!(index.get().await.1, 400); } From 6a742ee62c250a0108ef6706c9c52b462b3a2c8f Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 19:08:19 +0100 Subject: [PATCH 184/522] restore version route --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/routes/stats.rs | 6 +++++- meilisearch-http/tests/common/server.rs | 4 ++++ meilisearch-http/tests/integration.rs | 1 + meilisearch-http/tests/stats/mod.rs | 12 ++++++++++++ 6 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 meilisearch-http/tests/stats/mod.rs diff --git a/Cargo.lock b/Cargo.lock index bcdcec71c..dd0531ae3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.17.0" +version = "0.21.0-alpha" dependencies = [ "actix-cors", "actix-http 3.0.0-beta.4", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 9ae7baae2..f417d4391 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "MeiliSearch HTTP server" edition = "2018" license = "MIT" name = "meilisearch-http" -version = "0.17.0" +version = "0.21.0-alpha" [[bin]] name = "meilisearch" path = "src/main.rs" diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index eee6536c6..172b5dde9 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -56,5 +56,9 @@ struct VersionResponse { #[get("/version", wrap = "Authentication::Private")] async fn get_version() -> HttpResponse { - todo!() + HttpResponse::Ok().json(VersionResponse { + commit_sha: env!("VERGEN_SHA").to_string(), + build_date: env!("VERGEN_BUILD_TIMESTAMP").to_string(), + pkg_version: env!("CARGO_PKG_VERSION").to_string(), + }) } diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 6c7225a8f..43caf1dc6 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -69,4 +69,8 @@ impl Server { pub async fn list_indexes(&self) -> (Value, StatusCode) { self.service.get("/indexes").await } + + pub async fn version(&self) -> (Value, StatusCode) { + self.service.get("/version").await + } } diff --git a/meilisearch-http/tests/integration.rs b/meilisearch-http/tests/integration.rs index 7578a8dc1..8acc75ff9 100644 --- a/meilisearch-http/tests/integration.rs +++ b/meilisearch-http/tests/integration.rs @@ -4,6 +4,7 @@ mod index; mod search; mod settings; mod updates; +mod stats; // Tests are isolated by features in different modules to allow better readability, test // targetability, and improved incremental compilation times. diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs new file mode 100644 index 000000000..733e2ba79 --- /dev/null +++ b/meilisearch-http/tests/stats/mod.rs @@ -0,0 +1,12 @@ +use crate::common::Server; + +#[actix_rt::test] +async fn get_settings_unexisting_index() { + let server = Server::new().await; + let (response, code) = server.version().await; + assert_eq!(code, 200); + let version = response.as_object().unwrap(); + assert!(version.get("commitSha").is_some()); + assert!(version.get("buildDate").is_some()); + assert!(version.get("pkgVersion").is_some()); +} From f4cf96915a54334edc944894a9d28aeccdbf6510 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 16 Mar 2021 12:04:32 +0100 Subject: [PATCH 185/522] remove guard on add documetn route --- meilisearch-http/src/routes/document.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 59bc5fe0e..a5c071ac3 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -17,6 +17,7 @@ const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; macro_rules! guard_content_type { ($fn_name:ident, $guard_value:literal) => { + #[allow(dead_code)] fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { if let Some(content_type) = head.headers.get("Content-Type") { content_type @@ -44,7 +45,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(get_document) .service(delete_document) .service(get_all_documents) - .service(add_documents_json) + .service(add_documents) .service(update_documents) .service(delete_documents) .service(clear_all_documents); @@ -144,9 +145,8 @@ struct UpdateDocumentsQuery { #[post( "/indexes/{index_uid}/documents", wrap = "Authentication::Private", - guard = "guard_json" )] -async fn add_documents_json( +async fn add_documents( data: web::Data, path: web::Path, params: web::Query, @@ -201,7 +201,6 @@ async fn update_documents_default( #[put( "/indexes/{index_uid}/documents", wrap = "Authentication::Private", - guard = "guard_json" )] async fn update_documents( data: web::Data, From 9dd1ecdc2ae7be2b876ebae361ceaff915a7addf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 16 Mar 2021 10:26:08 +0100 Subject: [PATCH 186/522] Add bors configuration --- .github/workflows/rust.yml | 10 ++++++++-- bors.toml | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 bors.toml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ea4b41a6a..230339c49 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,12 +1,18 @@ name: Rust -on: push +on: + pull_request: + push: + # trying and staging branches are for Bors config + branches: + - trying + - staging env: CARGO_TERM_COLOR: always jobs: - build: + tests: runs-on: ubuntu-latest diff --git a/bors.toml b/bors.toml new file mode 100644 index 000000000..dd19e1f5b --- /dev/null +++ b/bors.toml @@ -0,0 +1,3 @@ +status = ['tests'] +# 3 hours timeout +timeout-sec = 10800 From 4aaa561147392ef4ab9dbc6b223d6050d449d825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 9 Mar 2021 16:43:14 +0100 Subject: [PATCH 187/522] Add release drafter file --- .github/release-draft-template.yml | 14 ++++++++++++++ .github/workflows/release-drafter.yml | 17 +++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 .github/release-draft-template.yml create mode 100644 .github/workflows/release-drafter.yml diff --git a/.github/release-draft-template.yml b/.github/release-draft-template.yml new file mode 100644 index 000000000..d61e1a84b --- /dev/null +++ b/.github/release-draft-template.yml @@ -0,0 +1,14 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +version-template: '0.21.0-alpha.$PATCH' +version-resolver: + default: patch +categories: + - title: 'Breaking changes ⚠️' + label: 'breaking-change' +template: | + ## Changes + + $CHANGES +no-changes-template: 'Changes are coming soon 😎' +sort-direction: 'ascending' diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 000000000..184de7258 --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,17 @@ +name: Release Drafter + +on: + push: + branches: + # - main + - release-drafter + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - uses: release-drafter/release-drafter@v5 + with: + config-name: release-draft-template.yml + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }} From 204c743bcc825e2f9aaa2b502d57de3020ed2969 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 16 Mar 2021 14:09:01 +0100 Subject: [PATCH 188/522] add json payload check on document addition --- Cargo.lock | 7 ++++ meilisearch-http/Cargo.toml | 1 + .../src/index_controller/update_actor.rs | 38 +++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcdcec71c..85ebb9c02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1823,6 +1823,7 @@ dependencies = [ "milli", "mime", "once_cell", + "oxidized-json-checker", "parking_lot", "rand 0.7.3", "rayon", @@ -2122,6 +2123,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "oxidized-json-checker" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938464aebf563f48ab86d1cfc0e2df952985c0b814d3108f41d1b85e7f5b0dac" + [[package]] name = "page_size" version = "0.4.2" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 9ae7baae2..e61ecf647 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -61,6 +61,7 @@ tempfile = "3.1.0" thiserror = "1.0.24" tokio = { version = "1", features = ["full"] } uuid = "0.8.2" +oxidized-json-checker = "0.3.2" [dependencies.sentry] default-features = false diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 1e788ff34..abf2ab8bc 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -1,13 +1,15 @@ use std::collections::{hash_map::Entry, HashMap}; +use std::io::SeekFrom; use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use super::index_actor::IndexActorHandle; use log::info; +use oxidized_json_checker::JsonChecker; +use super::index_actor::IndexActorHandle; use thiserror::Error; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; +use tokio::fs::OpenOptions; +use tokio::io::{AsyncWriteExt, AsyncSeekExt}; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; @@ -125,7 +127,11 @@ where let update_store = self.store.get_or_create(uuid).await?; let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); - let mut file = File::create(&path) + let mut file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&path) .await .map_err(|e| UpdateError::Error(Box::new(e)))?; @@ -146,7 +152,31 @@ where .await .map_err(|e| UpdateError::Error(Box::new(e)))?; + file.seek(SeekFrom::Start(0)) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))?; + + let mut file = file.into_std().await; + tokio::task::spawn_blocking(move || { + use std::io::{BufReader, sink, copy, Seek}; + + // If the payload is empty, ignore the check. + if file.metadata().map_err(|e| UpdateError::Error(Box::new(e)))?.len() > 0 { + // Check that the json payload is valid: + let reader = BufReader::new(&mut file); + let mut checker = JsonChecker::new(reader); + + if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() { + // The json file is invalid, we use Serde to get a nice error message: + file.seek(SeekFrom::Start(0)) + .map_err(|e| UpdateError::Error(Box::new(e)))?; + let _: serde_json::Value = serde_json::from_reader(file) + .map_err(|e| UpdateError::Error(Box::new(e)))?; + } + } + + // The payload is valid, we can register it to the update store. update_store .register_update(meta, path, uuid) .map(UpdateStatus::Pending) From 85f3b192d5c49c9199f7c3ab942efcb464b91b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 9 Mar 2021 19:04:32 +0100 Subject: [PATCH 189/522] Update release-draft-template.yml --- .github/release-draft-template.yml | 7 ++----- .github/workflows/release-drafter.yml | 5 ++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/release-draft-template.yml b/.github/release-draft-template.yml index d61e1a84b..e15489af8 100644 --- a/.github/release-draft-template.yml +++ b/.github/release-draft-template.yml @@ -1,14 +1,11 @@ name-template: 'v$RESOLVED_VERSION' tag-template: 'v$RESOLVED_VERSION' version-template: '0.21.0-alpha.$PATCH' -version-resolver: - default: patch -categories: - - title: 'Breaking changes ⚠️' - label: 'breaking-change' template: | ## Changes $CHANGES no-changes-template: 'Changes are coming soon 😎' sort-direction: 'ascending' +version-resolver: + default: patch diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 184de7258..dfcc162e1 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -3,8 +3,7 @@ name: Release Drafter on: push: branches: - # - main - - release-drafter + - main jobs: update_release_draft: @@ -12,6 +11,6 @@ jobs: steps: - uses: release-drafter/release-drafter@v5 with: - config-name: release-draft-template.yml + config-name: release-draft-template-ll.yml env: GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }} From 13c5289ff1190c75718a4ababad77e6c8ee41d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 16 Mar 2021 14:46:08 +0100 Subject: [PATCH 190/522] Update release-drafter.yml --- .github/workflows/release-drafter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index dfcc162e1..9ec8b9d64 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: release-drafter/release-drafter@v5 with: - config-name: release-draft-template-ll.yml + config-name: release-draft-template.yml env: GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }} From 761c2b0639a84ed3c8edae6faa81e90a1652f6c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 16 Mar 2021 15:16:33 +0100 Subject: [PATCH 191/522] Update release-draft-template.yml --- .github/release-draft-template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/release-draft-template.yml b/.github/release-draft-template.yml index e15489af8..33a8d7cb8 100644 --- a/.github/release-draft-template.yml +++ b/.github/release-draft-template.yml @@ -1,6 +1,8 @@ name-template: 'v$RESOLVED_VERSION' tag-template: 'v$RESOLVED_VERSION' version-template: '0.21.0-alpha.$PATCH' +exclude-labels: + - 'skip-changelog' template: | ## Changes From 3c25ab0d508efbc204daa4d725225ac983257882 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 16 Mar 2021 16:09:14 +0100 Subject: [PATCH 192/522] replace body with json --- meilisearch-http/src/routes/document.rs | 61 ++++++--------------- meilisearch-http/src/routes/index.rs | 44 +++++---------- meilisearch-http/src/routes/search.rs | 16 ++---- meilisearch-http/src/routes/settings/mod.rs | 43 ++++----------- 4 files changed, 46 insertions(+), 118 deletions(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index a5c071ac3..8e337d275 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -65,12 +65,9 @@ async fn get_document( .retrieve_document(index, id, None as Option>) .await { - Ok(document) => { - let json = serde_json::to_string(&document).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(document) => Ok(HttpResponse::Ok().json(document)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -87,12 +84,9 @@ async fn delete_document( .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]) .await { - Ok(result) => { - let json = serde_json::to_string(&result).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(result) => Ok(HttpResponse::Ok().json(result)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -125,12 +119,9 @@ async fn get_all_documents( ) .await { - Ok(docs) => { - let json = serde_json::to_string(&docs).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(docs) => Ok(HttpResponse::Ok().json(docs)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -142,10 +133,7 @@ struct UpdateDocumentsQuery { } /// Route used when the payload type is "application/json" -#[post( - "/indexes/{index_uid}/documents", - wrap = "Authentication::Private", -)] +#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn add_documents( data: web::Data, path: web::Path, @@ -163,13 +151,9 @@ async fn add_documents( .await; match addition_result { - Ok(update) => { - let value = serde_json::to_string(&update).unwrap(); - let response = HttpResponse::Ok().body(value); - Ok(response) - } + Ok(update) => Ok(HttpResponse::Ok().json(update)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -198,10 +182,7 @@ async fn update_documents_default( todo!() } -#[put( - "/indexes/{index_uid}/documents", - wrap = "Authentication::Private", -)] +#[put("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn update_documents( data: web::Data, path: web::Path, @@ -219,13 +200,9 @@ async fn update_documents( .await; match addition_result { - Ok(update) => { - let value = serde_json::to_string(&update).unwrap(); - let response = HttpResponse::Ok().body(value); - Ok(response) - } + Ok(update) => Ok(HttpResponse::Ok().json(update)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -249,12 +226,9 @@ async fn delete_documents( .collect(); match data.delete_documents(path.index_uid.clone(), ids).await { - Ok(result) => { - let json = serde_json::to_string(&result).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(result) => Ok(HttpResponse::Ok().json(result)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -265,12 +239,9 @@ async fn clear_all_documents( path: web::Path, ) -> Result { match data.clear_documents(path.index_uid.clone()).await { - Ok(update) => { - let json = serde_json::to_string(&update).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(update) => Ok(HttpResponse::Ok().json(update)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 29ba3d0d8..d5fac5170 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -21,12 +21,9 @@ pub fn services(cfg: &mut web::ServiceConfig) { #[get("/indexes", wrap = "Authentication::Private")] async fn list_indexes(data: web::Data) -> Result { match data.list_indexes().await { - Ok(indexes) => { - let json = serde_json::to_string(&indexes).unwrap(); - Ok(HttpResponse::Ok().body(&json)) - } + Ok(indexes) => Ok(HttpResponse::Ok().json(indexes)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -37,12 +34,9 @@ async fn get_index( path: web::Path, ) -> Result { match data.index(path.index_uid.clone()).await { - Ok(meta) => { - let json = serde_json::to_string(&meta).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(meta) => Ok(HttpResponse::Ok().json(meta)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -61,12 +55,9 @@ async fn create_index( ) -> Result { let body = body.into_inner(); match data.create_index(body.uid, body.primary_key).await { - Ok(meta) => { - let json = serde_json::to_string(&meta).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(meta) => Ok(HttpResponse::Ok().json(meta)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -99,12 +90,9 @@ async fn update_index( .update_index(path.into_inner().index_uid, body.primary_key, body.uid) .await { - Ok(meta) => { - let json = serde_json::to_string(&meta).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(meta) => Ok(HttpResponse::Ok().json(meta)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -117,7 +105,7 @@ async fn delete_index( match data.delete_index(path.index_uid.clone()).await { Ok(_) => Ok(HttpResponse::NoContent().finish()), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -141,12 +129,9 @@ async fn get_update_status( .get_update_status(params.index_uid, params.update_id) .await; match result { - Ok(meta) => { - let json = serde_json::to_string(&meta).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(meta) => Ok(HttpResponse::Ok().json(meta)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -158,12 +143,9 @@ async fn get_all_updates_status( ) -> Result { let result = data.get_updates_status(path.into_inner().index_uid).await; match result { - Ok(metas) => { - let json = serde_json::to_string(&metas).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(metas) => Ok(HttpResponse::Ok().json(metas)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index c98a5ac51..ae2caeb30 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -81,18 +81,15 @@ async fn search_with_url_query( Ok(q) => q, Err(e) => { return Ok( - HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() })) + HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() })) ) } }; let search_result = data.search(path.into_inner().index_uid, query).await; match search_result { - Ok(docs) => { - let docs = serde_json::to_string(&docs).unwrap(); - Ok(HttpResponse::Ok().body(docs)) - } + Ok(docs) => Ok(HttpResponse::Ok().json(docs)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -107,12 +104,9 @@ async fn search_with_post( .search(path.into_inner().index_uid, params.into_inner()) .await; match search_result { - Ok(docs) => { - let docs = serde_json::to_string(&docs).unwrap(); - Ok(HttpResponse::Ok().body(docs)) - } + Ok(docs) => Ok(HttpResponse::Ok().json(docs)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 5a6246f8c..6a7bfce48 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -27,12 +27,9 @@ macro_rules! make_setting_route { ..Default::default() }; match data.update_settings(index_uid.into_inner(), settings, false).await { - Ok(update_status) => { - let json = serde_json::to_string(&update_status).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(update_status) => Ok(HttpResponse::Ok().json(update_status)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -49,12 +46,9 @@ macro_rules! make_setting_route { }; match data.update_settings(index_uid.into_inner(), settings, true).await { - Ok(update_status) => { - let json = serde_json::to_string(&update_status).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(update_status) => Ok(HttpResponse::Ok().json(update_status)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -65,13 +59,9 @@ macro_rules! make_setting_route { index_uid: actix_web::web::Path, ) -> std::result::Result { match data.settings(index_uid.into_inner()).await { - Ok(settings) => { - let setting = settings.$attr; - let json = serde_json::to_string(&setting).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(settings) => Ok(HttpResponse::Ok().json(settings.$attr)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -141,12 +131,9 @@ async fn update_all( .update_settings(index_uid.into_inner(), body.into_inner(), true) .await { - Ok(update_result) => { - let json = serde_json::to_string(&update_result).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(update_result) => Ok(HttpResponse::Ok().json(update_result)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -157,12 +144,9 @@ async fn get_all( index_uid: web::Path, ) -> Result { match data.settings(index_uid.into_inner()).await { - Ok(settings) => { - let json = serde_json::to_string(&settings).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(settings) => Ok(HttpResponse::Ok().json(settings)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } @@ -177,12 +161,9 @@ async fn delete_all( .update_settings(index_uid.into_inner(), settings, false) .await { - Ok(update_result) => { - let json = serde_json::to_string(&update_result).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } + Ok(update_result) => Ok(HttpResponse::Ok().json(update_result)), Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } From a268d0e283625c26412d1320d855905cc5fba367 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 16 Mar 2021 17:42:01 +0100 Subject: [PATCH 193/522] return a 200 on health check --- meilisearch-http/src/routes/health.rs | 3 ++- meilisearch-http/tests/stats/mod.rs | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/health.rs b/meilisearch-http/src/routes/health.rs index c57e4c7e9..053790d61 100644 --- a/meilisearch-http/src/routes/health.rs +++ b/meilisearch-http/src/routes/health.rs @@ -9,5 +9,6 @@ pub fn services(cfg: &mut web::ServiceConfig) { #[get("/health")] async fn get_health() -> Result { - Ok(HttpResponse::NoContent().finish()) + let payload = serde_json::json!({ "status": "available" }); + Ok(HttpResponse::Ok().body(payload.to_string())) } diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index 733e2ba79..5e3c9db45 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -10,3 +10,12 @@ async fn get_settings_unexisting_index() { assert!(version.get("buildDate").is_some()); assert!(version.get("pkgVersion").is_some()); } + +#[actix_rt::test] +async fn test_healthyness() { + let server = Server::new().await; + + let (response, status_code) = server.service.get("/health").await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "available"); + } From 233c1e304dbf42dded214120dc4125ba8e01dc5c Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 16 Mar 2021 17:45:59 +0100 Subject: [PATCH 194/522] use json instead of body when crafting the request --- meilisearch-http/src/routes/health.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meilisearch-http/src/routes/health.rs b/meilisearch-http/src/routes/health.rs index 053790d61..8994df722 100644 --- a/meilisearch-http/src/routes/health.rs +++ b/meilisearch-http/src/routes/health.rs @@ -9,6 +9,5 @@ pub fn services(cfg: &mut web::ServiceConfig) { #[get("/health")] async fn get_health() -> Result { - let payload = serde_json::json!({ "status": "available" }); - Ok(HttpResponse::Ok().body(payload.to_string())) + Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) } From 8a52979ffab2757b76984330ed56675940c2f66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 16 Mar 2021 19:54:34 +0100 Subject: [PATCH 195/522] Update Cargo.toml --- meilisearch-http/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 1760cf844..4b8ecd13d 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "MeiliSearch HTTP server" edition = "2018" license = "MIT" name = "meilisearch-http" -version = "0.21.0-alpha" +version = "0.21.0-alpha.1" [[bin]] name = "meilisearch" path = "src/main.rs" From c8b05712fa1701e1980bc3bd1101f7c9c0b7fb2f Mon Sep 17 00:00:00 2001 From: Irevoire Date: Wed, 17 Mar 2021 14:44:32 +0100 Subject: [PATCH 196/522] return 202 on settings update / reset --- meilisearch-http/src/routes/settings/mod.rs | 4 ++-- meilisearch-http/tests/settings/get_settings.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 6a7bfce48..ee9b3e325 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -131,7 +131,7 @@ async fn update_all( .update_settings(index_uid.into_inner(), body.into_inner(), true) .await { - Ok(update_result) => Ok(HttpResponse::Ok().json(update_result)), + Ok(update_result) => Ok(HttpResponse::Accepted().json(update_result)), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -161,7 +161,7 @@ async fn delete_all( .update_settings(index_uid.into_inner(), settings, false) .await { - Ok(update_result) => Ok(HttpResponse::Ok().json(update_result)), + Ok(update_result) => Ok(HttpResponse::Accepted().json(update_result)), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index e4c38cb00..ac1d9df53 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -101,7 +101,7 @@ async fn update_setting_unexisting_index() { let server = Server::new().await; let index = server.index("test"); let (_response, code) = index.update_settings(json!({})).await; - assert_eq!(code, 200); + assert_eq!(code, 202); let (_response, code) = index.get().await; assert_eq!(code, 200); } From 6b4ea7f5942c496280d62c77661375c60c94e608 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Wed, 17 Mar 2021 15:09:13 +0100 Subject: [PATCH 197/522] ensure the reset_settings also return a 202 --- meilisearch-http/tests/settings/get_settings.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index ac1d9df53..1cbfdbb1d 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -104,6 +104,8 @@ async fn update_setting_unexisting_index() { assert_eq!(code, 202); let (_response, code) = index.get().await; assert_eq!(code, 200); + let (_response, code) = index.delete_settings().await; + assert_eq!(code, 202); } #[actix_rt::test] From 8b99860e850e2423a5e54a97af241cfd9280d140 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Mar 2021 08:27:53 +0100 Subject: [PATCH 198/522] use write sender for updates --- Cargo.lock | 2 +- meilisearch-http/src/index_controller/index_actor.rs | 6 +++--- meilisearch-http/tests/settings/get_settings.rs | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 141f6906c..dfb58ff89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.21.0-alpha" +version = "0.21.0-alpha.1" dependencies = [ "actix-cors", "actix-http 3.0.0-beta.4", diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index d9ec8fcda..cc6d67528 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -280,7 +280,7 @@ impl IndexActor { meta: Processing, data: File, ) -> Result { - debug!("Processing update {}", meta.id()); + log::info!("Processing update {}", meta.id()); let uuid = meta.index_uuid(); let update_handler = self.update_handler.clone(); let index = match self.store.get(*uuid).await? { @@ -443,7 +443,7 @@ impl IndexActorHandle { ) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Update { ret, meta, data }; - let _ = self.read_sender.send(msg).await; + let _ = self.write_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -500,7 +500,7 @@ impl IndexActorHandle { pub async fn delete(&self, uuid: Uuid) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Delete { uuid, ret }; - let _ = self.read_sender.send(msg).await; + let _ = self.write_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 1cbfdbb1d..b1030aea9 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -19,7 +19,6 @@ async fn get_settings() { assert_eq!(settings.keys().len(), 4); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); - println!("{:?}", settings); assert_eq!(settings["attributesForFaceting"], json!({})); assert_eq!( settings["rankingRules"], From 147756750b89f1c9a0113b158d41df30833992c0 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 18 Mar 2021 09:09:26 +0100 Subject: [PATCH 199/522] create uuid on successful update addition also change resolve to get in uuid resolver --- Cargo.lock | 2 +- meilisearch-http/src/index_controller/mod.rs | 114 ++++++++++-------- .../src/index_controller/uuid_resolver.rs | 73 ++++++----- 3 files changed, 107 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 141f6906c..dfb58ff89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.21.0-alpha" +version = "0.21.0-alpha.1" dependencies = [ "actix-cors", "actix-http 3.0.0-beta.4", diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 90c67e2b6..e8263ae68 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -16,10 +16,12 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::time::sleep; +use uuid::Uuid; use crate::index::{Document, SearchQuery, SearchResult}; use crate::index::{Facets, Settings, UpdateResult}; pub use updates::{Failed, Processed, Processing}; +use uuid_resolver::UuidError; pub type UpdateStatus = updates::UpdateStatus; @@ -83,38 +85,49 @@ impl IndexController { mut payload: Payload, primary_key: Option, ) -> anyhow::Result { - let uuid = self.uuid_resolver.get_or_create(uid).await?; - let meta = UpdateMeta::DocumentsAddition { - method, - format, - primary_key, - }; - let (sender, receiver) = mpsc::channel(10); + let perform_udpate = |uuid| async move { + let meta = UpdateMeta::DocumentsAddition { + method, + format, + primary_key, + }; + let (sender, receiver) = mpsc::channel(10); - // It is necessary to spawn a local task to senf the payload to the update handle to - // prevent dead_locking between the update_handle::update that waits for the update to be - // registered and the update_actor that waits for the the payload to be sent to it. - tokio::task::spawn_local(async move { - while let Some(bytes) = payload.next().await { - match bytes { - Ok(bytes) => { - let _ = sender.send(Ok(bytes)).await; - } - Err(e) => { - let error: Box = Box::new(e); - let _ = sender.send(Err(error)).await; + // It is necessary to spawn a local task to senf the payload to the update handle to + // prevent dead_locking between the update_handle::update that waits for the update to be + // registered and the update_actor that waits for the the payload to be sent to it. + tokio::task::spawn_local(async move { + while let Some(bytes) = payload.next().await { + match bytes { + Ok(bytes) => { + let _ = sender.send(Ok(bytes)).await; + } + Err(e) => { + let error: Box = Box::new(e); + let _ = sender.send(Err(error)).await; + } } } - } - }); + }); - // This must be done *AFTER* spawning the task. - let status = self.update_handle.update(meta, receiver, uuid).await?; - Ok(status) + // This must be done *AFTER* spawning the task. + self.update_handle.update(meta, receiver, uuid).await + }; + + match self.uuid_resolver.get(uid).await { + Ok(uuid) => Ok(perform_udpate(uuid).await?), + Err(UuidError::UnexistingIndex(name)) => { + let uuid = Uuid::new_v4(); + let status = perform_udpate(uuid).await?; + self.uuid_resolver.insert(name, uuid).await?; + Ok(status) + } + Err(e) => Err(e.into()), + } } pub async fn clear_documents(&self, uid: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(uid).await?; + let uuid = self.uuid_resolver.get(uid).await?; let meta = UpdateMeta::ClearDocuments; let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; @@ -126,7 +139,7 @@ impl IndexController { uid: String, document_ids: Vec, ) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(uid).await?; + let uuid = self.uuid_resolver.get(uid).await?; let meta = UpdateMeta::DeleteDocuments; let (sender, receiver) = mpsc::channel(10); @@ -146,26 +159,23 @@ impl IndexController { settings: Settings, create: bool, ) -> anyhow::Result { - let uuid = if create { - let uuid = self.uuid_resolver.get_or_create(uid).await?; - // We need to create the index upfront, since it would otherwise only be created when - // the update is processed. This would make calls to GET index to fail until the update - // is complete. Since this is get or create, we ignore the error when the index already - // exists. - match self.index_handle.create_index(uuid, None).await { - Ok(_) | Err(index_actor::IndexError::IndexAlreadyExists) => (), - Err(e) => return Err(e.into()), - } - uuid - } else { - self.uuid_resolver.resolve(uid).await? + let perform_udpate = |uuid| async move { + let meta = UpdateMeta::Settings(settings); + // Nothing so send, drop the sender right away, as not to block the update actor. + let (_, receiver) = mpsc::channel(1); + self.update_handle.update(meta, receiver, uuid).await }; - let meta = UpdateMeta::Settings(settings); - // Nothing so send, drop the sender right away, as not to block the update actor. - let (_, receiver) = mpsc::channel(1); - let status = self.update_handle.update(meta, receiver, uuid).await?; - Ok(status) + match self.uuid_resolver.get(uid).await { + Ok(uuid) => Ok(perform_udpate(uuid).await?), + Err(UuidError::UnexistingIndex(name)) if create => { + let uuid = Uuid::new_v4(); + let status = perform_udpate(uuid).await?; + self.uuid_resolver.insert(name, uuid).await?; + Ok(status) + } + Err(e) => Err(e.into()), + } } pub async fn create_index( @@ -190,13 +200,13 @@ impl IndexController { } pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(uid).await?; + let uuid = self.uuid_resolver.get(uid).await?; let result = self.update_handle.update_status(uuid, id).await?; Ok(result) } pub async fn all_update_status(&self, uid: String) -> anyhow::Result> { - let uuid = self.uuid_resolver.resolve(uid).await?; + let uuid = self.uuid_resolver.get(uid).await?; let result = self.update_handle.get_all_updates_status(uuid).await?; Ok(result) } @@ -216,7 +226,7 @@ impl IndexController { } pub async fn settings(&self, uid: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let uuid = self.uuid_resolver.get(uid.clone()).await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) } @@ -228,7 +238,7 @@ impl IndexController { limit: usize, attributes_to_retrieve: Option>, ) -> anyhow::Result> { - let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let uuid = self.uuid_resolver.get(uid.clone()).await?; let documents = self .index_handle .documents(uuid, offset, limit, attributes_to_retrieve) @@ -242,7 +252,7 @@ impl IndexController { doc_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let uuid = self.uuid_resolver.get(uid.clone()).await?; let document = self .index_handle .document(uuid, doc_id, attributes_to_retrieve) @@ -259,20 +269,20 @@ impl IndexController { bail!("Can't change the index uid.") } - let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let uuid = self.uuid_resolver.get(uid.clone()).await?; let meta = self.index_handle.update_index(uuid, index_settings).await?; let meta = IndexMetadata { name: uid.clone(), uid, meta }; Ok(meta) } pub async fn search(&self, uid: String, query: SearchQuery) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(uid).await?; + let uuid = self.uuid_resolver.get(uid).await?; let result = self.index_handle.search(uuid, query).await?; Ok(result) } pub async fn get_index(&self, uid: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let uuid = self.uuid_resolver.get(uid.clone()).await?; let meta = self.index_handle.get_index_meta(uuid).await?; let meta = IndexMetadata { name: uid.clone(), uid, meta }; Ok(meta) diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 2ee9c6b17..328080d90 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -13,11 +13,7 @@ pub type Result = std::result::Result; #[derive(Debug)] enum UuidResolveMsg { - Resolve { - uid: String, - ret: oneshot::Sender>, - }, - GetOrCreate { + Get { uid: String, ret: oneshot::Sender>, }, @@ -32,6 +28,11 @@ enum UuidResolveMsg { List { ret: oneshot::Sender>>, }, + Insert { + uuid: Uuid, + name: String, + ret: oneshot::Sender>, + } } struct UuidResolverActor { @@ -54,11 +55,8 @@ impl UuidResolverActor { Some(Create { uid: name, ret }) => { let _ = ret.send(self.handle_create(name).await); } - Some(GetOrCreate { uid: name, ret }) => { - let _ = ret.send(self.handle_get_or_create(name).await); - } - Some(Resolve { uid: name, ret }) => { - let _ = ret.send(self.handle_resolve(name).await); + Some(Get { uid: name, ret }) => { + let _ = ret.send(self.handle_get(name).await); } Some(Delete { uid: name, ret }) => { let _ = ret.send(self.handle_delete(name).await); @@ -66,6 +64,9 @@ impl UuidResolverActor { Some(List { ret }) => { let _ = ret.send(self.handle_list().await); } + Some(Insert { ret, uuid, name }) => { + let _ = ret.send(self.handle_insert(name, uuid).await); + } // all senders have been dropped, need to quit. None => break, } @@ -81,14 +82,7 @@ impl UuidResolverActor { self.store.create_uuid(uid, true).await } - async fn handle_get_or_create(&self, uid: String) -> Result { - if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)); - } - self.store.create_uuid(uid, false).await - } - - async fn handle_resolve(&self, uid: String) -> Result { + async fn handle_get(&self, uid: String) -> Result { self.store .get_uuid(uid.clone()) .await? @@ -106,6 +100,14 @@ impl UuidResolverActor { let result = self.store.list().await?; Ok(result) } + + async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> { + if !is_index_uid_valid(&uid) { + return Err(UuidError::BadlyFormatted(uid)); + } + self.store.insert(uid, uuid).await?; + Ok(()) + } } fn is_index_uid_valid(uid: &str) -> bool { @@ -127,18 +129,9 @@ impl UuidResolverHandle { Ok(Self { sender }) } - pub async fn resolve(&self, name: String) -> anyhow::Result { + pub async fn get(&self, name: String) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Resolve { uid: name, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - - pub async fn get_or_create(&self, name: String) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::GetOrCreate { uid: name, ret }; + let msg = UuidResolveMsg::Get { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -171,6 +164,15 @@ impl UuidResolverHandle { .await .expect("Uuid resolver actor has been killed")?) } + + pub async fn insert(&self, name: String, uuid: Uuid) -> anyhow::Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Insert { ret, name, uuid }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } } #[derive(Debug, Error)] @@ -197,6 +199,7 @@ trait UuidStore { async fn get_uuid(&self, uid: String) -> Result>; async fn delete(&self, uid: String) -> Result>; async fn list(&self) -> Result>; + async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; } struct HeedUuidStore { @@ -292,4 +295,16 @@ impl UuidStore for HeedUuidStore { }) .await? } + + async fn insert(&self, name: String, uuid: Uuid) -> Result<()> { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + let mut txn = env.write_txn()?; + db.put(&mut txn, &name, uuid.as_bytes())?; + txn.commit()?; + Ok(()) + }) + .await? + } } From 7ecefe37da9307c2e55a1ce3b177fe4f61dabb1d Mon Sep 17 00:00:00 2001 From: Irevoire Date: Fri, 19 Mar 2021 11:34:54 +0100 Subject: [PATCH 200/522] fix root route --- meilisearch-http/src/lib.rs | 2 +- meilisearch-http/src/routes/mod.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index cdd874a35..9532273aa 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -44,7 +44,7 @@ macro_rules! create_app { let app = if $enable_frontend { app.service(load_html).service(load_css) } else { - app + app.service(running) }; app.wrap( Cors::default() diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index e1345fa08..9164f2d2f 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -29,6 +29,7 @@ impl IndexUpdateResponse { } } +/// Return the dashboard, should not be used in production. See [running] #[get("/")] pub async fn load_html() -> HttpResponse { HttpResponse::Ok() @@ -36,6 +37,17 @@ pub async fn load_html() -> HttpResponse { .body(include_str!("../../public/interface.html").to_string()) } +/// Always return a 200 with: +/// ```json +/// { +/// "status": "Meilisearch is running" +/// } +/// ``` +#[get("/")] +pub async fn running() -> HttpResponse { + HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" })) +} + #[get("/bulma.min.css")] pub async fn load_css() -> HttpResponse { HttpResponse::Ok() From 91089db444098a1d26d9cd2529fc0f588553db5b Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 22 Mar 2021 18:41:33 +0100 Subject: [PATCH 201/522] add the exhaustive nb hits to be ISO, currently it's always set to false --- meilisearch-http/src/index/search.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index a0de632be..9b7739456 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -41,6 +41,7 @@ pub struct SearchQuery { pub struct SearchResult { pub hits: Vec>, pub nb_hits: u64, + pub exhaustive_nb_hits: bool, // currently this field only exist to be ISO and is always alse pub query: String, pub limit: usize, pub offset: usize, @@ -107,6 +108,7 @@ impl Index { }; let result = SearchResult { + exhaustive_nb_hits: false, // not implemented, we use it to be ISO hits: documents, nb_hits, query: query.q.clone().unwrap_or_default(), From b690f1103a83a5467f9a2d3b211e0c7488369839 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Mar 2021 19:25:56 +0100 Subject: [PATCH 202/522] fix typos --- meilisearch-http/src/index_controller/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index e8263ae68..10c6e8f41 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -85,7 +85,7 @@ impl IndexController { mut payload: Payload, primary_key: Option, ) -> anyhow::Result { - let perform_udpate = |uuid| async move { + let perform_update = |uuid| async move { let meta = UpdateMeta::DocumentsAddition { method, format, @@ -93,7 +93,7 @@ impl IndexController { }; let (sender, receiver) = mpsc::channel(10); - // It is necessary to spawn a local task to senf the payload to the update handle to + // It is necessary to spawn a local task to send the payload to the update handle to // prevent dead_locking between the update_handle::update that waits for the update to be // registered and the update_actor that waits for the the payload to be sent to it. tokio::task::spawn_local(async move { @@ -115,10 +115,10 @@ impl IndexController { }; match self.uuid_resolver.get(uid).await { - Ok(uuid) => Ok(perform_udpate(uuid).await?), + Ok(uuid) => Ok(perform_update(uuid).await?), Err(UuidError::UnexistingIndex(name)) => { let uuid = Uuid::new_v4(); - let status = perform_udpate(uuid).await?; + let status = perform_update(uuid).await?; self.uuid_resolver.insert(name, uuid).await?; Ok(status) } From 5f33672f0eef019e62e5ccb59f9ee8bf4a8f32c3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Mar 2021 19:49:21 +0100 Subject: [PATCH 203/522] change payload send to use stream methods --- meilisearch-http/src/index_controller/mod.rs | 47 +++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 10c6e8f41..2bd373a5c 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -82,7 +82,7 @@ impl IndexController { uid: String, method: milli::update::IndexDocumentsMethod, format: milli::update::UpdateFormat, - mut payload: Payload, + payload: Payload, primary_key: Option, ) -> anyhow::Result { let perform_update = |uuid| async move { @@ -97,17 +97,16 @@ impl IndexController { // prevent dead_locking between the update_handle::update that waits for the update to be // registered and the update_actor that waits for the the payload to be sent to it. tokio::task::spawn_local(async move { - while let Some(bytes) = payload.next().await { - match bytes { - Ok(bytes) => { - let _ = sender.send(Ok(bytes)).await; - } - Err(e) => { - let error: Box = Box::new(e); - let _ = sender.send(Err(error)).await; - } - } - } + payload + .map(|bytes| { + bytes.map_err(|e| { + Box::new(e) as Box + }) + }) + .for_each(|r| async { + let _ = sender.send(r).await; + }) + .await }); // This must be done *AFTER* spawning the task. @@ -187,7 +186,11 @@ impl IndexController { let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; let _ = self.update_handle.create(uuid).await?; - let meta = IndexMetadata { name: uid.clone(), uid, meta }; + let meta = IndexMetadata { + name: uid.clone(), + uid, + meta, + }; Ok(meta) } @@ -218,7 +221,11 @@ impl IndexController { for (uid, uuid) in uuids { let meta = self.index_handle.get_index_meta(uuid).await?; - let meta = IndexMetadata { name: uid.clone(), uid, meta }; + let meta = IndexMetadata { + name: uid.clone(), + uid, + meta, + }; ret.push(meta); } @@ -271,7 +278,11 @@ impl IndexController { let uuid = self.uuid_resolver.get(uid.clone()).await?; let meta = self.index_handle.update_index(uuid, index_settings).await?; - let meta = IndexMetadata { name: uid.clone(), uid, meta }; + let meta = IndexMetadata { + name: uid.clone(), + uid, + meta, + }; Ok(meta) } @@ -284,7 +295,11 @@ impl IndexController { pub async fn get_index(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.get(uid.clone()).await?; let meta = self.index_handle.get_index_meta(uuid).await?; - let meta = IndexMetadata { name: uid.clone(), uid, meta }; + let meta = IndexMetadata { + name: uid.clone(), + uid, + meta, + }; Ok(meta) } } From 46d7cedb1890786e1be7a5d22ea7b297d28becaf Mon Sep 17 00:00:00 2001 From: Irevoire Date: Tue, 23 Mar 2021 10:46:59 +0100 Subject: [PATCH 204/522] Update meilisearch-http/src/index/search.rs Co-authored-by: marin --- meilisearch-http/src/index/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 9b7739456..12c05919e 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -41,7 +41,7 @@ pub struct SearchQuery { pub struct SearchResult { pub hits: Vec>, pub nb_hits: u64, - pub exhaustive_nb_hits: bool, // currently this field only exist to be ISO and is always alse + pub exhaustive_nb_hits: bool, // currently this field only exist to be ISO and is always false pub query: String, pub limit: usize, pub offset: usize, From cc81aca6a47a7cc299bff5c4733e4c9c534ef46f Mon Sep 17 00:00:00 2001 From: Irevoire Date: Tue, 23 Mar 2021 10:47:19 +0100 Subject: [PATCH 205/522] Update meilisearch-http/src/index/search.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clémentine Urquizar --- meilisearch-http/src/index/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 12c05919e..670014cdf 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -108,7 +108,7 @@ impl Index { }; let result = SearchResult { - exhaustive_nb_hits: false, // not implemented, we use it to be ISO + exhaustive_nb_hits: false, // not implemented yet hits: documents, nb_hits, query: query.q.clone().unwrap_or_default(), From 127e94486610d1143bd7e71ab01c94a3380451e8 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Tue, 23 Mar 2021 19:13:22 +0100 Subject: [PATCH 206/522] Update meilisearch-http/src/index/search.rs Co-authored-by: marin --- meilisearch-http/src/index/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 670014cdf..2002266a1 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -41,7 +41,7 @@ pub struct SearchQuery { pub struct SearchResult { pub hits: Vec>, pub nb_hits: u64, - pub exhaustive_nb_hits: bool, // currently this field only exist to be ISO and is always false + pub exhaustive_nb_hits: bool, pub query: String, pub limit: usize, pub offset: usize, From ee838be41b7eac8c0629c30d84388c149d74f6ac Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 17 Mar 2021 11:53:23 +0100 Subject: [PATCH 207/522] implement snapshot scheduler --- meilisearch-http/src/index_controller/mod.rs | 16 +++++-- .../src/index_controller/snapshot.rs | 48 +++++++++++++++++++ .../src/{snapshot.rs => snapshot_old.rs} | 12 ++--- 3 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 meilisearch-http/src/index_controller/snapshot.rs rename meilisearch-http/src/{snapshot.rs => snapshot_old.rs} (97%) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 90c67e2b6..0449941ce 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -4,6 +4,7 @@ mod update_handler; mod update_store; mod updates; mod uuid_resolver; +mod snapshot; use std::path::Path; use std::sync::Arc; @@ -19,7 +20,9 @@ use tokio::time::sleep; use crate::index::{Document, SearchQuery, SearchResult}; use crate::index::{Facets, Settings, UpdateResult}; + pub use updates::{Failed, Processed, Processing}; +use snapshot::SnapshotService; pub type UpdateStatus = updates::UpdateStatus; @@ -65,12 +68,19 @@ impl IndexController { update_store_size: usize, ) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; - let index_actor = index_actor::IndexActorHandle::new(&path, index_size)?; + let index_handle = index_actor::IndexActorHandle::new(&path, index_size)?; let update_handle = - update_actor::UpdateActorHandle::new(index_actor.clone(), &path, update_store_size)?; + update_actor::UpdateActorHandle::new(index_handle.clone(), &path, update_store_size)?; + let snapshot_service = SnapshotService::new( + index_handle.clone(), + uuid_resolver.clone(), + update_handle.clone(), + Duration::from_millis(10000), + "/dev/toto".into()); + tokio::task::spawn(snapshot_service.run()); Ok(Self { uuid_resolver, - index_handle: index_actor, + index_handle, update_handle, }) } diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs new file mode 100644 index 000000000..6d77941bb --- /dev/null +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -0,0 +1,48 @@ +use std::path::PathBuf; +use std::time::Duration; + +use tokio::time::interval; + +use super::index_actor::IndexActorHandle; +use super::update_actor::UpdateActorHandle; +use super::uuid_resolver::UuidResolverHandle; + +#[allow(dead_code)] +pub struct SnapshotService { + index_handle: IndexActorHandle, + uuid_resolver_handle: UuidResolverHandle, + update_handle: UpdateActorHandle, + snapshot_period: Duration, + snapshot_path: PathBuf, +} + +impl SnapshotService { + pub fn new( + index_handle: IndexActorHandle, + uuid_resolver_handle: UuidResolverHandle, + update_handle: UpdateActorHandle, + snapshot_period: Duration, + snapshot_path: PathBuf, + ) -> Self { + Self { + index_handle, + uuid_resolver_handle, + update_handle, + snapshot_period, + snapshot_path, + } + } + + pub async fn run(self) { + let mut interval = interval(self.snapshot_period); + + loop { + interval.tick().await; + self.perform_snapshot().await; + } + } + + async fn perform_snapshot(&self) { + println!("performing snapshot in {:?}", self.snapshot_path); + } +} diff --git a/meilisearch-http/src/snapshot.rs b/meilisearch-http/src/snapshot_old.rs similarity index 97% rename from meilisearch-http/src/snapshot.rs rename to meilisearch-http/src/snapshot_old.rs index 520044f84..ed5cd9a81 100644 --- a/meilisearch-http/src/snapshot.rs +++ b/meilisearch-http/src/snapshot_old.rs @@ -37,14 +37,14 @@ pub fn create_snapshot(data: &Data, snapshot_path: &Path) -> Result<(), Error> { } pub fn schedule_snapshot(data: Data, snapshot_dir: &Path, time_gap_s: u64) -> Result<(), Error> { - if snapshot_dir.file_name().is_none() { + if snapshot_dir.file_name().is_none() { return Err(Error::Internal("invalid snapshot file path".to_string())); } let db_name = Path::new(&data.db_path).file_name().ok_or_else(|| Error::Internal("invalid database name".to_string()))?; create_dir_all(snapshot_dir)?; let snapshot_path = snapshot_dir.join(format!("{}.snapshot", db_name.to_str().unwrap_or("data.ms"))); - - thread::spawn(move || loop { + + thread::spawn(move || loop { if let Err(e) = create_snapshot(&data, &snapshot_path) { error!("Unsuccessful snapshot creation: {}", e); } @@ -72,12 +72,12 @@ mod tests { let file_1_relative = Path::new("file1.txt"); let subdir_relative = Path::new("subdir/"); let file_2_relative = Path::new("subdir/file2.txt"); - + create_dir_all(src_dir.join(subdir_relative)).unwrap(); fs::File::create(src_dir.join(file_1_relative)).unwrap().write_all(b"Hello_file_1").unwrap(); fs::File::create(src_dir.join(file_2_relative)).unwrap().write_all(b"Hello_file_2").unwrap(); - + assert!(compression::to_tar_gz(&src_dir, &archive_path).is_ok()); assert!(archive_path.exists()); assert!(load_snapshot(&dest_dir.to_str().unwrap(), &archive_path, false, false).is_ok()); @@ -89,7 +89,7 @@ mod tests { let contents = fs::read_to_string(dest_dir.join(file_1_relative)).unwrap(); assert_eq!(contents, "Hello_file_1"); - + let contents = fs::read_to_string(dest_dir.join(file_2_relative)).unwrap(); assert_eq!(contents, "Hello_file_2"); } From c966b1dd94e3699e93c4037df4a4066b7eb4f8b1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 17 Mar 2021 12:01:56 +0100 Subject: [PATCH 208/522] use options to schedule snapshot --- meilisearch-http/src/data/mod.rs | 4 +-- meilisearch-http/src/index_controller/mod.rs | 27 +++++++++++++------- meilisearch-http/src/option.rs | 4 +-- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index c312c2912..19004da70 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -60,9 +60,7 @@ impl Data { let path = options.db_path.clone(); create_dir_all(&path)?; - let index_size = options.max_mdb_size.get_bytes() as usize; - let update_store_size = options.max_udb_size.get_bytes() as usize; - let index_controller = IndexController::new(&path, index_size, update_store_size)?; + let index_controller = IndexController::new(&path, &options)?; let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 0449941ce..fc0fd3d46 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -20,6 +20,7 @@ use tokio::time::sleep; use crate::index::{Document, SearchQuery, SearchResult}; use crate::index::{Facets, Settings, UpdateResult}; +use crate::option::Opt; pub use updates::{Failed, Processed, Processing}; use snapshot::SnapshotService; @@ -64,20 +65,28 @@ pub struct IndexController { impl IndexController { pub fn new( path: impl AsRef, - index_size: usize, - update_store_size: usize, + options: &Opt, ) -> anyhow::Result { + let index_size = options.max_mdb_size.get_bytes() as usize; + let update_store_size = options.max_udb_size.get_bytes() as usize; + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; let index_handle = index_actor::IndexActorHandle::new(&path, index_size)?; let update_handle = update_actor::UpdateActorHandle::new(index_handle.clone(), &path, update_store_size)?; - let snapshot_service = SnapshotService::new( - index_handle.clone(), - uuid_resolver.clone(), - update_handle.clone(), - Duration::from_millis(10000), - "/dev/toto".into()); - tokio::task::spawn(snapshot_service.run()); + + if options.schedule_snapshot { + let snapshot_service = SnapshotService::new( + index_handle.clone(), + uuid_resolver.clone(), + update_handle.clone(), + Duration::from_secs(options.snapshot_interval_sec), + options.snapshot_dir.clone() + ); + + tokio::task::spawn(snapshot_service.run()); + } + Ok(Self { uuid_resolver, index_handle, diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 82eb75fc1..1997718cc 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -191,8 +191,8 @@ pub struct Opt { pub schedule_snapshot: bool, /// Defines time interval, in seconds, between each snapshot creation. - #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC")] - pub snapshot_interval_sec: Option, + #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h + pub snapshot_interval_sec: u64, /// Folder where dumps are created when the dump route is called. #[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")] From 35a7b800ebcde401cf2e4e3db97d53b4104d5589 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 19 Mar 2021 20:08:00 +0100 Subject: [PATCH 209/522] snapshot indexes --- .../src/index_controller/index_actor.rs | 54 ++++++++++++++++++- .../src/index_controller/snapshot.rs | 17 ++++-- .../src/index_controller/update_actor.rs | 27 ++++++++++ .../src/index_controller/uuid_resolver.rs | 45 +++++++++++++++- 4 files changed, 137 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index cc6d67528..9c8f58a62 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -8,7 +8,7 @@ use async_stream::stream; use chrono::{DateTime, Utc}; use futures::pin_mut; use futures::stream::StreamExt; -use heed::EnvOpenOptions; +use heed::{CompactionOption, EnvOpenOptions}; use log::debug; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -103,6 +103,11 @@ enum IndexMsg { index_settings: IndexSettings, ret: oneshot::Sender>, }, + Snapshot { + uuids: Vec, + path: PathBuf, + ret: oneshot::Sender>, + } } struct IndexActor { @@ -251,6 +256,9 @@ impl IndexActor { } => { let _ = ret.send(self.handle_update_index(uuid, index_settings).await); } + Snapshot { uuids, path, ret } => { + let _ = ret.send(self.handle_snapshot(uuids, path).await); + } } } @@ -403,6 +411,39 @@ impl IndexActor { .await .map_err(|e| IndexError::Error(e.into()))? } + + async fn handle_snapshot(&self, uuids: Vec, mut path: PathBuf) -> Result<()> { + use tokio::fs::create_dir_all; + + path.push("indexes"); + println!("performing index snapshot in {:?}", path); + create_dir_all(&path) + .await + .map_err(|e| IndexError::Error(e.into()))?; + + let mut handles = Vec::new(); + for uuid in uuids { + if let Some(index) = self.store.get(uuid).await? { + let index_path = path.join(format!("index-{}", uuid)); + let handle = spawn_blocking(move || -> anyhow::Result<()> { + // Get write txn to wait for ongoing write transaction before snapshot. + let _txn = index.write_txn()?; + index.env.copy_to_path(index_path, CompactionOption::Enabled)?; + Ok(()) + }); + handles.push(handle); + } + } + + for handle in handles { + handle + .await + .map_err(|e| IndexError::Error(e.into()))? + .map_err(|e| IndexError::Error(e.into()))?; + } + + Ok(()) + } } #[derive(Clone)] @@ -525,6 +566,17 @@ impl IndexActorHandle { let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Snapshot { + uuids, + path, + ret, + }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct HeedIndexStore { diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 6d77941bb..85b39f506 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -1,7 +1,9 @@ use std::path::PathBuf; use std::time::Duration; +use std::fs::create_dir_all; use tokio::time::interval; +use uuid::Uuid; use super::index_actor::IndexActorHandle; use super::update_actor::UpdateActorHandle; @@ -38,11 +40,20 @@ impl SnapshotService { loop { interval.tick().await; - self.perform_snapshot().await; + self.perform_snapshot().await.unwrap(); } } - async fn perform_snapshot(&self) { - println!("performing snapshot in {:?}", self.snapshot_path); + async fn perform_snapshot(&self) -> anyhow::Result<()> { + let temp_snapshot_path = self + .snapshot_path + .join(format!("tmp-{}", Uuid::new_v4())); + create_dir_all(&temp_snapshot_path)?; + let uuids = self.uuid_resolver_handle.snapshot(temp_snapshot_path.clone()).await?; + let index_snapshot = self.index_handle.snapshot(uuids.clone(), temp_snapshot_path.clone()); + let updates_snapshot = self.update_handle.snapshot(uuids.clone(), temp_snapshot_path.clone()); + let (first, second) = tokio::join!(updates_snapshot, index_snapshot); + println!("results: {:?}, {:?}", first, second); + Ok(()) } } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index abf2ab8bc..6caba133b 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -55,6 +55,11 @@ enum UpdateMsg { uuid: Uuid, ret: oneshot::Sender>, }, + Snapshot { + uuids: Vec, + path: PathBuf, + ret: oneshot::Sender>, + } } struct UpdateActor { @@ -113,6 +118,9 @@ where Some(Create { uuid, ret }) => { let _ = ret.send(self.handle_create(uuid).await); } + Some(Snapshot { uuids, path, ret }) => { + let _ = ret.send(self.handle_snapshot(uuids, path).await); + } None => break, } } @@ -232,6 +240,16 @@ where let _ = self.store.get_or_create(uuid).await?; Ok(()) } + + async fn handle_snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { + use tokio::time; + use std::time::Duration; + + println!("performing update snapshot"); + time::sleep(Duration::from_secs(2)).await; + println!("Update snapshot done"); + Ok(()) + } } #[derive(Clone)] @@ -274,7 +292,9 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } +} +impl UpdateActorHandle { pub async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::ListUpdates { uuid, ret }; @@ -302,6 +322,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Snapshot { uuids, path, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 2ee9c6b17..c31d776b3 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -1,4 +1,5 @@ -use std::{fs::create_dir_all, path::Path}; +use std::fs::create_dir_all; +use std::path::{Path, PathBuf}; use heed::{ types::{ByteSlice, Str}, @@ -8,6 +9,7 @@ use log::{info, warn}; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; +use heed::CompactionOption; pub type Result = std::result::Result; @@ -32,6 +34,10 @@ enum UuidResolveMsg { List { ret: oneshot::Sender>>, }, + SnapshotRequest { + path: PathBuf, + ret: oneshot::Sender>>, + }, } struct UuidResolverActor { @@ -66,6 +72,9 @@ impl UuidResolverActor { Some(List { ret }) => { let _ = ret.send(self.handle_list().await); } + Some(SnapshotRequest { path, ret }) => { + let _ = ret.send(self.handle_snapshot(path).await); + } // all senders have been dropped, need to quit. None => break, } @@ -106,6 +115,10 @@ impl UuidResolverActor { let result = self.store.list().await?; Ok(result) } + + async fn handle_snapshot(&self, path: PathBuf) -> Result> { + self.store.snapshot(path).await + } } fn is_index_uid_valid(uid: &str) -> bool { @@ -171,6 +184,15 @@ impl UuidResolverHandle { .await .expect("Uuid resolver actor has been killed")?) } + + pub async fn snapshot(&self, path: PathBuf) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::SnapshotRequest { path, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } } #[derive(Debug, Error)] @@ -197,6 +219,7 @@ trait UuidStore { async fn get_uuid(&self, uid: String) -> Result>; async fn delete(&self, uid: String) -> Result>; async fn list(&self) -> Result>; + async fn snapshot(&self, path: PathBuf) -> Result>; } struct HeedUuidStore { @@ -242,7 +265,6 @@ impl UuidStore for HeedUuidStore { }) .await? } - async fn get_uuid(&self, name: String) -> Result> { let env = self.env.clone(); let db = self.db; @@ -292,4 +314,23 @@ impl UuidStore for HeedUuidStore { }) .await? } + + async fn snapshot(&self, mut path: PathBuf) -> Result> { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + // Write transaction to acquire a lock on the database. + let txn = env.write_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (_, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push(uuid) + } + path.push("uuids"); + env.copy_to_path(path, CompactionOption::Enabled)?; + Ok(entries) + }) + .await? + } } From 520f7c09bafd17fd7d285510f850e8334e850d3f Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 20 Mar 2021 11:50:57 +0100 Subject: [PATCH 210/522] sequential index snapshot --- .../src/index_controller/index_actor.rs | 39 ++++----- .../src/index_controller/snapshot.rs | 7 +- .../src/index_controller/update_actor.rs | 80 +++++++++++++------ .../src/index_controller/update_store.rs | 2 +- 4 files changed, 75 insertions(+), 53 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 9c8f58a62..bfb7127f3 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -104,7 +104,7 @@ enum IndexMsg { ret: oneshot::Sender>, }, Snapshot { - uuids: Vec, + uuid: Uuid, path: PathBuf, ret: oneshot::Sender>, } @@ -256,8 +256,8 @@ impl IndexActor { } => { let _ = ret.send(self.handle_update_index(uuid, index_settings).await); } - Snapshot { uuids, path, ret } => { - let _ = ret.send(self.handle_snapshot(uuids, path).await); + Snapshot { uuid, path, ret } => { + let _ = ret.send(self.handle_snapshot(uuid, path).await); } } } @@ -412,7 +412,7 @@ impl IndexActor { .map_err(|e| IndexError::Error(e.into()))? } - async fn handle_snapshot(&self, uuids: Vec, mut path: PathBuf) -> Result<()> { + async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> { use tokio::fs::create_dir_all; path.push("indexes"); @@ -421,25 +421,14 @@ impl IndexActor { .await .map_err(|e| IndexError::Error(e.into()))?; - let mut handles = Vec::new(); - for uuid in uuids { - if let Some(index) = self.store.get(uuid).await? { - let index_path = path.join(format!("index-{}", uuid)); - let handle = spawn_blocking(move || -> anyhow::Result<()> { - // Get write txn to wait for ongoing write transaction before snapshot. - let _txn = index.write_txn()?; - index.env.copy_to_path(index_path, CompactionOption::Enabled)?; - Ok(()) - }); - handles.push(handle); - } - } - - for handle in handles { - handle - .await - .map_err(|e| IndexError::Error(e.into()))? - .map_err(|e| IndexError::Error(e.into()))?; + if let Some(index) = self.store.get(uuid).await? { + let index_path = path.join(format!("index-{}", uuid)); + spawn_blocking(move || -> anyhow::Result<()> { + // Get write txn to wait for ongoing write transaction before snapshot. + let _txn = index.write_txn()?; + index.env.copy_to_path(index_path, CompactionOption::Enabled)?; + Ok(()) + }); } Ok(()) @@ -567,10 +556,10 @@ impl IndexActorHandle { Ok(receiver.await.expect("IndexActor has been killed")?) } - pub async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { + pub async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Snapshot { - uuids, + uuid, path, ret, }; diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 85b39f506..75f6c1f82 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -50,10 +50,9 @@ impl SnapshotService { .join(format!("tmp-{}", Uuid::new_v4())); create_dir_all(&temp_snapshot_path)?; let uuids = self.uuid_resolver_handle.snapshot(temp_snapshot_path.clone()).await?; - let index_snapshot = self.index_handle.snapshot(uuids.clone(), temp_snapshot_path.clone()); - let updates_snapshot = self.update_handle.snapshot(uuids.clone(), temp_snapshot_path.clone()); - let (first, second) = tokio::join!(updates_snapshot, index_snapshot); - println!("results: {:?}, {:?}", first, second); + for uuid in uuids { + self.update_handle.snapshot(uuid, temp_snapshot_path.clone()).await?; + } Ok(()) } } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 6caba133b..6e017dcf5 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -1,15 +1,16 @@ use std::collections::{hash_map::Entry, HashMap}; -use std::io::SeekFrom; use std::fs::{create_dir_all, remove_dir_all}; +use std::io::SeekFrom; use std::path::{Path, PathBuf}; use std::sync::Arc; +use super::index_actor::IndexActorHandle; +use heed::CompactionOption; use log::info; use oxidized_json_checker::JsonChecker; -use super::index_actor::IndexActorHandle; use thiserror::Error; use tokio::fs::OpenOptions; -use tokio::io::{AsyncWriteExt, AsyncSeekExt}; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; @@ -56,16 +57,17 @@ enum UpdateMsg { ret: oneshot::Sender>, }, Snapshot { - uuids: Vec, + uuid: Uuid, path: PathBuf, ret: oneshot::Sender>, - } + }, } struct UpdateActor { path: PathBuf, store: S, inbox: mpsc::Receiver>, + index_handle: IndexActorHandle, } #[async_trait::async_trait] @@ -84,11 +86,17 @@ where store: S, inbox: mpsc::Receiver>, path: impl AsRef, + index_handle: IndexActorHandle, ) -> anyhow::Result { - let path = path.as_ref().to_owned().join("update_files"); - create_dir_all(&path)?; + let path = path.as_ref().to_owned(); + create_dir_all(path.join("update_files"))?; assert!(path.exists()); - Ok(Self { store, inbox, path }) + Ok(Self { + store, + inbox, + path, + index_handle, + }) } async fn run(mut self) { @@ -118,8 +126,8 @@ where Some(Create { uuid, ret }) => { let _ = ret.send(self.handle_create(uuid).await); } - Some(Snapshot { uuids, path, ret }) => { - let _ = ret.send(self.handle_snapshot(uuids, path).await); + Some(Snapshot { uuid, path, ret }) => { + let _ = ret.send(self.handle_snapshot(uuid, path).await); } None => break, } @@ -134,7 +142,9 @@ where ) -> Result { let update_store = self.store.get_or_create(uuid).await?; let update_file_id = uuid::Uuid::new_v4(); - let path = self.path.join(format!("update_{}", update_file_id)); + let path = self + .path + .join(format!("update_files/update_{}", update_file_id)); let mut file = OpenOptions::new() .read(true) .write(true) @@ -167,10 +177,15 @@ where let mut file = file.into_std().await; tokio::task::spawn_blocking(move || { - use std::io::{BufReader, sink, copy, Seek}; + use std::io::{copy, sink, BufReader, Seek}; // If the payload is empty, ignore the check. - if file.metadata().map_err(|e| UpdateError::Error(Box::new(e)))?.len() > 0 { + if file + .metadata() + .map_err(|e| UpdateError::Error(Box::new(e)))? + .len() + > 0 + { // Check that the json payload is valid: let reader = BufReader::new(&mut file); let mut checker = JsonChecker::new(reader); @@ -241,13 +256,32 @@ where Ok(()) } - async fn handle_snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { - use tokio::time; - use std::time::Duration; + async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + use tokio::fs; + + let update_path = path.join("updates"); + fs::create_dir_all(&update_path) + .await + .map_err(|e| UpdateError::Error(e.into()))?; + + let index_handle = self.index_handle.clone(); + if let Some(update_store) = self.store.get(uuid).await? { + let snapshot_path = update_path.join(format!("update-{}", uuid)); + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let _txn = update_store.env.write_txn()?; + update_store + .env + .copy_to_path(&snapshot_path, CompactionOption::Enabled)?; + futures::executor::block_on( + async move { index_handle.snapshot(uuid, path).await }, + )?; + Ok(()) + }) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?; + } - println!("performing update snapshot"); - time::sleep(Duration::from_secs(2)).await; - println!("Update snapshot done"); Ok(()) } } @@ -268,8 +302,8 @@ where ) -> anyhow::Result { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); - let store = MapUpdateStoreStore::new(index_handle, &path, update_store_size); - let actor = UpdateActor::new(store, receiver, path)?; + let store = MapUpdateStoreStore::new(index_handle.clone(), &path, update_store_size); + let actor = UpdateActor::new(store, receiver, path, index_handle)?; tokio::task::spawn(actor.run()); @@ -323,9 +357,9 @@ impl UpdateActorHandle { receiver.await.expect("update actor killed.") } - pub async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { + pub async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Snapshot { uuids, path, ret }; + let msg = UpdateMsg::Snapshot { uuid, path, ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index 6de30ab7f..5280ed94e 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -16,7 +16,7 @@ type BEU64 = heed::zerocopy::U64; #[derive(Clone)] pub struct UpdateStore { - env: Env, + pub env: Env, pending_meta: Database, SerdeJson>>, pending: Database, SerdeJson>, processed_meta: Database, SerdeJson>>, From 7f6a54cb12fe1eaf781482e6ff76d71019b89a86 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 20 Mar 2021 17:24:08 +0100 Subject: [PATCH 211/522] add lock to prevent snapshot during update --- .../src/index_controller/snapshot.rs | 1 + .../src/index_controller/update_actor.rs | 21 ++++------ .../src/index_controller/update_store.rs | 40 +++++++++++++++++-- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 75f6c1f82..8e26fbfac 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -52,6 +52,7 @@ impl SnapshotService { let uuids = self.uuid_resolver_handle.snapshot(temp_snapshot_path.clone()).await?; for uuid in uuids { self.update_handle.snapshot(uuid, temp_snapshot_path.clone()).await?; + println!("performed snapshot for index {}", uuid); } Ok(()) } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 6e017dcf5..64eab5221 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -5,7 +5,6 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use super::index_actor::IndexActorHandle; -use heed::CompactionOption; use log::info; use oxidized_json_checker::JsonChecker; use thiserror::Error; @@ -257,21 +256,17 @@ where } async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { - use tokio::fs; - - let update_path = path.join("updates"); - fs::create_dir_all(&update_path) - .await - .map_err(|e| UpdateError::Error(e.into()))?; - let index_handle = self.index_handle.clone(); if let Some(update_store) = self.store.get(uuid).await? { - let snapshot_path = update_path.join(format!("update-{}", uuid)); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - let _txn = update_store.env.write_txn()?; - update_store - .env - .copy_to_path(&snapshot_path, CompactionOption::Enabled)?; + // acquire write lock to prevent further writes during snapshot + // the update lock must be acquired BEFORE the write lock to prevent dead lock + let _lock = update_store.update_lock.lock(); + let mut txn = update_store.env.write_txn()?; + + // create db snapshot + update_store.snapshot(&mut txn, &path, uuid)?; + futures::executor::block_on( async move { index_handle.snapshot(uuid, path).await }, )?; diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index 5280ed94e..587b060af 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -1,10 +1,10 @@ -use std::fs::remove_file; +use std::fs::{remove_file, create_dir_all, copy}; use std::path::{Path, PathBuf}; use std::sync::Arc; use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; -use heed::{Database, Env, EnvOpenOptions}; -use parking_lot::RwLock; +use heed::{Database, Env, EnvOpenOptions, CompactionOption}; +use parking_lot::{RwLock, Mutex}; use serde::{Deserialize, Serialize}; use std::fs::File; use tokio::sync::mpsc; @@ -24,6 +24,9 @@ pub struct UpdateStore { aborted_meta: Database, SerdeJson>>, processing: Arc>>>, notification_sender: mpsc::Sender<()>, + /// A lock on the update loop. This is meant to prevent a snapshot to occur while an update is + /// processing, while not preventing writes all together during an update + pub update_lock: Arc>, } pub trait HandleUpdate { @@ -76,6 +79,8 @@ where // Send a first notification to trigger the process. let _ = notification_sender.send(()); + let update_lock = Arc::new(Mutex::new(())); + let update_store = Arc::new(UpdateStore { env, pending, @@ -85,6 +90,7 @@ where notification_sender, failed_meta, processing, + update_lock, }); // We need a weak reference so we can take ownership on the arc later when we @@ -190,6 +196,7 @@ where where U: HandleUpdate, { + let _lock = self.update_lock.lock(); // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; let first_meta = self.pending_meta.first(&rtxn)?; @@ -371,6 +378,33 @@ where Ok(aborted_updates) } + + pub fn snapshot(&self, txn: &mut heed::RwTxn, path: impl AsRef, uuid: Uuid) -> anyhow::Result<()> { + println!("snapshoting updates in {:?}", path.as_ref()); + let update_path = path.as_ref().join("updates"); + create_dir_all(&update_path)?; + + let snapshot_path = update_path.join(format!("update-{}", uuid)); + // acquire write lock to prevent further writes during snapshot + println!("acquired lock"); + + // create db snapshot + self.env.copy_to_path(&snapshot_path, CompactionOption::Enabled)?; + + let update_files_path = update_path.join("update_files"); + create_dir_all(&update_files_path)?; + + for path in self.pending.iter(&txn)? { + let (_, path) = path?; + let name = path.file_name().unwrap(); + let to = update_files_path.join(name); + copy(path, to)?; + } + + println!("done"); + + Ok(()) + } } //#[cfg(test)] From 4847884165f235847ac1c5d9d296abe7419aa15c Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Mar 2021 10:17:38 +0100 Subject: [PATCH 212/522] restore snapshots --- meilisearch-http/src/helpers/compression.rs | 8 +-- .../src/index_controller/index_actor.rs | 29 ++++++-- meilisearch-http/src/index_controller/mod.rs | 69 +++++++++++++++---- .../src/index_controller/update_actor.rs | 44 ++++++++++-- .../src/index_controller/uuid_resolver.rs | 26 ++++++- 5 files changed, 143 insertions(+), 33 deletions(-) diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index bbf14d578..7e8b5e3f3 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -17,11 +17,11 @@ pub fn to_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> { Ok(()) } -pub fn from_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> { - let f = File::open(src)?; +pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Error> { + let f = File::open(&src)?; let gz = GzDecoder::new(f); let mut ar = Archive::new(gz); - create_dir_all(dest)?; - ar.unpack(dest)?; + create_dir_all(&dest)?; + ar.unpack(&dest)?; Ok(()) } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index bfb7127f3..4b5c68b56 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -26,6 +26,7 @@ use crate::index_controller::{ UpdateMeta, }; use crate::option::IndexerOpts; +use crate::helpers::compression; pub type Result = std::result::Result; type AsyncMap = Arc>>; @@ -107,7 +108,7 @@ enum IndexMsg { uuid: Uuid, path: PathBuf, ret: oneshot::Sender>, - } + }, } struct IndexActor { @@ -426,7 +427,9 @@ impl IndexActor { spawn_blocking(move || -> anyhow::Result<()> { // Get write txn to wait for ongoing write transaction before snapshot. let _txn = index.write_txn()?; - index.env.copy_to_path(index_path, CompactionOption::Enabled)?; + index + .env + .copy_to_path(index_path, CompactionOption::Enabled)?; Ok(()) }); } @@ -455,6 +458,22 @@ impl IndexActorHandle { }) } + pub fn from_snapshot( + path: impl AsRef, + index_size: usize, + snapshot_path: impl AsRef, + ) -> anyhow::Result { + let snapshot_path = snapshot_path.as_ref().join("indexes"); + let indexes_path = path.as_ref().join("indexes"); + for entry in snapshot_path.read_dir()? { + let entry = entry?; + let src = snapshot_path.join(entry.file_name()); + let dest = indexes_path.join(entry.file_name()); + compression::from_tar_gz(src, dest)?; + } + Self::new(path, index_size) + } + pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { @@ -558,11 +577,7 @@ impl IndexActorHandle { pub async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Snapshot { - uuid, - path, - ret, - }; + let msg = IndexMsg::Snapshot { uuid, path, ret }; let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index fc0fd3d46..d2d11d44b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -1,10 +1,10 @@ mod index_actor; +mod snapshot; mod update_actor; mod update_handler; mod update_store; mod updates; mod uuid_resolver; -mod snapshot; use std::path::Path; use std::sync::Arc; @@ -22,8 +22,8 @@ use crate::index::{Document, SearchQuery, SearchResult}; use crate::index::{Facets, Settings, UpdateResult}; use crate::option::Opt; -pub use updates::{Failed, Processed, Processing}; use snapshot::SnapshotService; +pub use updates::{Failed, Processed, Processing}; pub type UpdateStatus = updates::UpdateStatus; @@ -63,17 +63,40 @@ pub struct IndexController { } impl IndexController { - pub fn new( - path: impl AsRef, - options: &Opt, - ) -> anyhow::Result { + pub fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { let index_size = options.max_mdb_size.get_bytes() as usize; let update_store_size = options.max_udb_size.get_bytes() as usize; - let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; - let index_handle = index_actor::IndexActorHandle::new(&path, index_size)?; - let update_handle = - update_actor::UpdateActorHandle::new(index_handle.clone(), &path, update_store_size)?; + let uuid_resolver; + let index_handle; + let update_handle; + + match options.import_snapshot { + Some(ref snapshot_path) => { + uuid_resolver = + uuid_resolver::UuidResolverHandle::from_snapshot(&path, &snapshot_path)?; + index_handle = index_actor::IndexActorHandle::from_snapshot( + &path, + index_size, + &snapshot_path, + )?; + update_handle = update_actor::UpdateActorHandle::from_snapshot( + index_handle.clone(), + &path, + update_store_size, + &snapshot_path, + )?; + } + None => { + uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; + index_handle = index_actor::IndexActorHandle::new(&path, index_size)?; + update_handle = update_actor::UpdateActorHandle::new( + index_handle.clone(), + &path, + update_store_size, + )?; + } + } if options.schedule_snapshot { let snapshot_service = SnapshotService::new( @@ -81,7 +104,7 @@ impl IndexController { uuid_resolver.clone(), update_handle.clone(), Duration::from_secs(options.snapshot_interval_sec), - options.snapshot_dir.clone() + options.snapshot_dir.clone(), ); tokio::task::spawn(snapshot_service.run()); @@ -196,7 +219,11 @@ impl IndexController { let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; let _ = self.update_handle.create(uuid).await?; - let meta = IndexMetadata { name: uid.clone(), uid, meta }; + let meta = IndexMetadata { + name: uid.clone(), + uid, + meta, + }; Ok(meta) } @@ -227,7 +254,11 @@ impl IndexController { for (uid, uuid) in uuids { let meta = self.index_handle.get_index_meta(uuid).await?; - let meta = IndexMetadata { name: uid.clone(), uid, meta }; + let meta = IndexMetadata { + name: uid.clone(), + uid, + meta, + }; ret.push(meta); } @@ -280,7 +311,11 @@ impl IndexController { let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let meta = self.index_handle.update_index(uuid, index_settings).await?; - let meta = IndexMetadata { name: uid.clone(), uid, meta }; + let meta = IndexMetadata { + name: uid.clone(), + uid, + meta, + }; Ok(meta) } @@ -293,7 +328,11 @@ impl IndexController { pub async fn get_index(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let meta = self.index_handle.get_index_meta(uuid).await?; - let meta = IndexMetadata { name: uid.clone(), uid, meta }; + let meta = IndexMetadata { + name: uid.clone(), + uid, + meta, + }; Ok(meta) } } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 64eab5221..9b2a90370 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -1,5 +1,5 @@ use std::collections::{hash_map::Entry, HashMap}; -use std::fs::{create_dir_all, remove_dir_all}; +use std::fs; use std::io::SeekFrom; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -16,6 +16,7 @@ use uuid::Uuid; use super::get_arc_ownership_blocking; use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; +use crate::helpers::compression; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; @@ -88,7 +89,7 @@ where index_handle: IndexActorHandle, ) -> anyhow::Result { let path = path.as_ref().to_owned(); - create_dir_all(path.join("update_files"))?; + fs::create_dir_all(path.join("update_files"))?; assert!(path.exists()); Ok(Self { store, @@ -305,6 +306,41 @@ where Ok(Self { sender }) } + pub fn from_snapshot( + index_handle: IndexActorHandle, + path: impl AsRef, + update_store_size: usize, + snapshot: impl AsRef, + ) -> anyhow::Result { + let src = snapshot.as_ref().join("updates"); + let dst = path.as_ref().join("updates"); + fs::create_dir_all(&dst)?; + + // restore the update stores + for entry in src.read_dir()? { + let entry = entry?; + // filter out the update_files directory. + if entry.file_type()?.is_file() { + let src = src.join(entry.file_name()); + let dest = dst.join(entry.file_name()); + compression::from_tar_gz(src, dest)?; + } + } + + // restore the update files + let src = src.join("update_files"); + let dst = dst.join("update_files"); + fs::create_dir_all(&dst)?; + for entry in src.read_dir()? { + let entry = entry?; + let src = src.join(entry.file_name()); + let dst = dst.join(entry.file_name()); + fs::copy(src, dst)?; + } + + Self::new(index_handle, path, update_store_size) + } + pub async fn update( &self, meta: UpdateMeta, @@ -393,7 +429,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { let update_store_size = self.update_store_size; options.map_size(update_store_size); let path = self.path.clone().join(format!("updates-{}", e.key())); - create_dir_all(&path).unwrap(); + fs::create_dir_all(&path).unwrap(); let index_handle = self.index_handle.clone(); let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle.update(meta, file)) @@ -448,7 +484,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { let store = self.db.write().await.remove(&uuid); let path = self.path.clone().join(format!("updates-{}", uuid)); if store.is_some() || path.exists() { - remove_dir_all(path).unwrap(); + fs::remove_dir_all(path).unwrap(); } Ok(store) } diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index c31d776b3..7d755e9b6 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -3,13 +3,16 @@ use std::path::{Path, PathBuf}; use heed::{ types::{ByteSlice, Str}, - Database, Env, EnvOpenOptions, + Database, Env, EnvOpenOptions,CompactionOption }; use log::{info, warn}; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use heed::CompactionOption; + +use crate::helpers::compression; + +const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB pub type Result = std::result::Result; @@ -140,6 +143,17 @@ impl UuidResolverHandle { Ok(Self { sender }) } + pub fn from_snapshot( + db_path: impl AsRef, + snapshot_path: impl AsRef + ) -> anyhow::Result { + let (sender, reveiver) = mpsc::channel(100); + let store = HeedUuidStore::from_snapshot(snapshot_path, db_path)?; + let actor = UuidResolverActor::new(reveiver, store); + tokio::spawn(actor.run()); + Ok(Self { sender }) + } + pub async fn resolve(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Resolve { uid: name, ret }; @@ -232,11 +246,17 @@ impl HeedUuidStore { let path = path.as_ref().join("index_uuids"); create_dir_all(&path)?; let mut options = EnvOpenOptions::new(); - options.map_size(1_073_741_824); // 1GB + options.map_size(UUID_STORE_SIZE); // 1GB let env = options.open(path)?; let db = env.create_database(None)?; Ok(Self { env, db }) } + + fn from_snapshot(snapshot: impl AsRef, path: impl AsRef) -> anyhow::Result { + let snapshot = snapshot.as_ref().join("uuids"); + compression::from_tar_gz(snapshot, &path)?; + Self::new(path) + } } #[async_trait::async_trait] From a85e7abb0c93a955a7880404a94234de604c8177 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Mar 2021 16:30:20 +0100 Subject: [PATCH 213/522] fix snapshot creation --- meilisearch-http/src/helpers/compression.rs | 2 ++ meilisearch-http/src/index_controller/index_actor.rs | 6 +++++- meilisearch-http/src/index_controller/update_store.rs | 6 +++--- meilisearch-http/src/index_controller/uuid_resolver.rs | 9 ++++++--- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index 7e8b5e3f3..e12e5b479 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -18,10 +18,12 @@ pub fn to_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> { } pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Error> { + println!("inflating from {:?} to {:?}", src.as_ref(), dest.as_ref()); let f = File::open(&src)?; let gz = GzDecoder::new(f); let mut ar = Archive::new(gz); create_dir_all(&dest)?; ar.unpack(&dest)?; + println!("here"); Ok(()) } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 4b5c68b56..bfa3097a7 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -423,7 +423,11 @@ impl IndexActor { .map_err(|e| IndexError::Error(e.into()))?; if let Some(index) = self.store.get(uuid).await? { - let index_path = path.join(format!("index-{}", uuid)); + let mut index_path = path.join(format!("index-{}", uuid)); + create_dir_all(&index_path) + .await + .map_err(|e| IndexError::Error(e.into()))?; + index_path.push("data.mdb"); spawn_blocking(move || -> anyhow::Result<()> { // Get write txn to wait for ongoing write transaction before snapshot. let _txn = index.write_txn()?; diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index 587b060af..e17c56e10 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -380,13 +380,13 @@ where } pub fn snapshot(&self, txn: &mut heed::RwTxn, path: impl AsRef, uuid: Uuid) -> anyhow::Result<()> { - println!("snapshoting updates in {:?}", path.as_ref()); let update_path = path.as_ref().join("updates"); create_dir_all(&update_path)?; - let snapshot_path = update_path.join(format!("update-{}", uuid)); + let mut snapshot_path = update_path.join(format!("update-{}", uuid)); // acquire write lock to prevent further writes during snapshot - println!("acquired lock"); + create_dir_all(&snapshot_path)?; + snapshot_path.push("data.mdb"); // create db snapshot self.env.copy_to_path(&snapshot_path, CompactionOption::Enabled)?; diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 7d755e9b6..5bdbe667f 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -253,8 +253,9 @@ impl HeedUuidStore { } fn from_snapshot(snapshot: impl AsRef, path: impl AsRef) -> anyhow::Result { - let snapshot = snapshot.as_ref().join("uuids"); - compression::from_tar_gz(snapshot, &path)?; + let src = snapshot.as_ref().join("uuids"); + let dst = path.as_ref().join("uuids"); + compression::from_tar_gz(src, dst)?; Self::new(path) } } @@ -347,7 +348,9 @@ impl UuidStore for HeedUuidStore { let uuid = Uuid::from_slice(uuid)?; entries.push(uuid) } - path.push("uuids"); + path.push("index_uuids"); + create_dir_all(&path).unwrap(); + path.push("data.mdb"); env.copy_to_path(path, CompactionOption::Enabled)?; Ok(entries) }) From 44dcfe29aa809bc50d42e5cc528dc1262e0932c3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Mar 2021 16:51:53 +0100 Subject: [PATCH 214/522] clean snapshot creation --- meilisearch-http/src/helpers/compression.rs | 2 +- .../src/index_controller/snapshot.rs | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index e12e5b479..4fc39a353 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -7,7 +7,7 @@ use tar::{Archive, Builder}; use crate::error::Error; -pub fn to_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> { +pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Error> { let f = File::create(dest)?; let gz_encoder = GzEncoder::new(f, Compression::default()); let mut tar_encoder = Builder::new(gz_encoder); diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 8e26fbfac..b2bbf17d2 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -1,10 +1,12 @@ use std::path::PathBuf; use std::time::Duration; -use std::fs::create_dir_all; +use anyhow::bail; +use tokio::fs; +use tokio::task::spawn_blocking; use tokio::time::interval; -use uuid::Uuid; +use crate::helpers::compression; use super::index_actor::IndexActorHandle; use super::update_actor::UpdateActorHandle; use super::uuid_resolver::UuidResolverHandle; @@ -45,15 +47,28 @@ impl SnapshotService { } async fn perform_snapshot(&self) -> anyhow::Result<()> { - let temp_snapshot_path = self - .snapshot_path - .join(format!("tmp-{}", Uuid::new_v4())); - create_dir_all(&temp_snapshot_path)?; + if self.snapshot_path.file_name().is_none() { + bail!("invalid snapshot file path"); + } + + let temp_snapshot_dir = spawn_blocking(move || tempfile::tempdir_in(".")).await??; + let temp_snapshot_path = temp_snapshot_dir.path().to_owned(); + + fs::create_dir_all(&temp_snapshot_path).await?; + let uuids = self.uuid_resolver_handle.snapshot(temp_snapshot_path.clone()).await?; for uuid in uuids { self.update_handle.snapshot(uuid, temp_snapshot_path.clone()).await?; - println!("performed snapshot for index {}", uuid); } + + let temp_snapshot_file = temp_snapshot_path.with_extension("temp"); + + let temp_snapshot_file_clone = temp_snapshot_file.clone(); + let temp_snapshot_path_clone = temp_snapshot_path.clone(); + spawn_blocking(move || compression::to_tar_gz(temp_snapshot_path_clone, temp_snapshot_file_clone)).await??; + + fs::rename(temp_snapshot_file, &self.snapshot_path).await?; + Ok(()) } } From d73fbdef2e2de107169978a5d15235a210d6ed12 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Mar 2021 16:58:19 +0100 Subject: [PATCH 215/522] remove from snapshot --- .../src/index_controller/index_actor.rs | 16 -------- meilisearch-http/src/index_controller/mod.rs | 37 ++++--------------- .../src/index_controller/update_actor.rs | 35 ------------------ .../src/index_controller/uuid_resolver.rs | 18 --------- 4 files changed, 7 insertions(+), 99 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index bfa3097a7..6474bdde4 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -462,22 +462,6 @@ impl IndexActorHandle { }) } - pub fn from_snapshot( - path: impl AsRef, - index_size: usize, - snapshot_path: impl AsRef, - ) -> anyhow::Result { - let snapshot_path = snapshot_path.as_ref().join("indexes"); - let indexes_path = path.as_ref().join("indexes"); - for entry in snapshot_path.read_dir()? { - let entry = entry?; - let src = snapshot_path.join(entry.file_name()); - let dest = indexes_path.join(entry.file_name()); - compression::from_tar_gz(src, dest)?; - } - Self::new(path, index_size) - } - pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index d2d11d44b..1f4e857d6 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -67,36 +67,13 @@ impl IndexController { let index_size = options.max_mdb_size.get_bytes() as usize; let update_store_size = options.max_udb_size.get_bytes() as usize; - let uuid_resolver; - let index_handle; - let update_handle; - - match options.import_snapshot { - Some(ref snapshot_path) => { - uuid_resolver = - uuid_resolver::UuidResolverHandle::from_snapshot(&path, &snapshot_path)?; - index_handle = index_actor::IndexActorHandle::from_snapshot( - &path, - index_size, - &snapshot_path, - )?; - update_handle = update_actor::UpdateActorHandle::from_snapshot( - index_handle.clone(), - &path, - update_store_size, - &snapshot_path, - )?; - } - None => { - uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; - index_handle = index_actor::IndexActorHandle::new(&path, index_size)?; - update_handle = update_actor::UpdateActorHandle::new( - index_handle.clone(), - &path, - update_store_size, - )?; - } - } + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; + let index_handle = index_actor::IndexActorHandle::new(&path, index_size)?; + let update_handle = update_actor::UpdateActorHandle::new( + index_handle.clone(), + &path, + update_store_size, + )?; if options.schedule_snapshot { let snapshot_service = SnapshotService::new( diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 9b2a90370..5cea8e64d 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -306,41 +306,6 @@ where Ok(Self { sender }) } - pub fn from_snapshot( - index_handle: IndexActorHandle, - path: impl AsRef, - update_store_size: usize, - snapshot: impl AsRef, - ) -> anyhow::Result { - let src = snapshot.as_ref().join("updates"); - let dst = path.as_ref().join("updates"); - fs::create_dir_all(&dst)?; - - // restore the update stores - for entry in src.read_dir()? { - let entry = entry?; - // filter out the update_files directory. - if entry.file_type()?.is_file() { - let src = src.join(entry.file_name()); - let dest = dst.join(entry.file_name()); - compression::from_tar_gz(src, dest)?; - } - } - - // restore the update files - let src = src.join("update_files"); - let dst = dst.join("update_files"); - fs::create_dir_all(&dst)?; - for entry in src.read_dir()? { - let entry = entry?; - let src = src.join(entry.file_name()); - let dst = dst.join(entry.file_name()); - fs::copy(src, dst)?; - } - - Self::new(index_handle, path, update_store_size) - } - pub async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 5bdbe667f..7cd41833c 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -143,17 +143,6 @@ impl UuidResolverHandle { Ok(Self { sender }) } - pub fn from_snapshot( - db_path: impl AsRef, - snapshot_path: impl AsRef - ) -> anyhow::Result { - let (sender, reveiver) = mpsc::channel(100); - let store = HeedUuidStore::from_snapshot(snapshot_path, db_path)?; - let actor = UuidResolverActor::new(reveiver, store); - tokio::spawn(actor.run()); - Ok(Self { sender }) - } - pub async fn resolve(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Resolve { uid: name, ret }; @@ -251,13 +240,6 @@ impl HeedUuidStore { let db = env.create_database(None)?; Ok(Self { env, db }) } - - fn from_snapshot(snapshot: impl AsRef, path: impl AsRef) -> anyhow::Result { - let src = snapshot.as_ref().join("uuids"); - let dst = path.as_ref().join("uuids"); - compression::from_tar_gz(src, dst)?; - Self::new(path) - } } #[async_trait::async_trait] From e9da191b7d840832bf8f76a384a3743d3861c3c6 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Mar 2021 19:19:37 +0100 Subject: [PATCH 216/522] fix snapshot bugs --- meilisearch-http/src/helpers/compression.rs | 13 ++++++------- .../src/index_controller/index_actor.rs | 7 ++++--- meilisearch-http/src/index_controller/mod.rs | 5 +++++ .../src/index_controller/snapshot.rs | 17 +++++++++++++---- .../src/index_controller/update_actor.rs | 1 - .../src/index_controller/uuid_resolver.rs | 14 ++++++++------ 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index 4fc39a353..74578ed0c 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -1,29 +1,28 @@ -use flate2::read::GzDecoder; -use flate2::write::GzEncoder; -use flate2::Compression; use std::fs::{create_dir_all, File}; +use std::io::Write; use std::path::Path; + +use flate2::{Compression, write::GzEncoder, read::GzDecoder}; use tar::{Archive, Builder}; use crate::error::Error; pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Error> { - let f = File::create(dest)?; - let gz_encoder = GzEncoder::new(f, Compression::default()); + let mut f = File::create(dest)?; + let gz_encoder = GzEncoder::new(&mut f, Compression::default()); let mut tar_encoder = Builder::new(gz_encoder); tar_encoder.append_dir_all(".", src)?; let gz_encoder = tar_encoder.into_inner()?; gz_encoder.finish()?; + f.flush()?; Ok(()) } pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Error> { - println!("inflating from {:?} to {:?}", src.as_ref(), dest.as_ref()); let f = File::open(&src)?; let gz = GzDecoder::new(f); let mut ar = Archive::new(gz); create_dir_all(&dest)?; ar.unpack(&dest)?; - println!("here"); Ok(()) } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 6474bdde4..9c4feef8a 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -26,7 +26,6 @@ use crate::index_controller::{ UpdateMeta, }; use crate::option::IndexerOpts; -use crate::helpers::compression; pub type Result = std::result::Result; type AsyncMap = Arc>>; @@ -417,7 +416,6 @@ impl IndexActor { use tokio::fs::create_dir_all; path.push("indexes"); - println!("performing index snapshot in {:?}", path); create_dir_all(&path) .await .map_err(|e| IndexError::Error(e.into()))?; @@ -435,7 +433,10 @@ impl IndexActor { .env .copy_to_path(index_path, CompactionOption::Enabled)?; Ok(()) - }); + }) + .await + .map_err(|e| IndexError::Error(e.into()))? + .map_err(|e| IndexError::Error(e.into()))?; } Ok(()) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 1f4e857d6..9fab9088a 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -21,6 +21,7 @@ use tokio::time::sleep; use crate::index::{Document, SearchQuery, SearchResult}; use crate::index::{Facets, Settings, UpdateResult}; use crate::option::Opt; +use crate::helpers::compression; use snapshot::SnapshotService; pub use updates::{Failed, Processed, Processing}; @@ -67,6 +68,10 @@ impl IndexController { let index_size = options.max_mdb_size.get_bytes() as usize; let update_store_size = options.max_udb_size.get_bytes() as usize; + if let Some(ref path) = options.import_snapshot { + compression::from_tar_gz(path, &options.db_path)?; + } + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; let index_handle = index_actor::IndexActorHandle::new(&path, index_size)?; let update_handle = update_actor::UpdateActorHandle::new( diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index b2bbf17d2..fc7ef7fed 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -2,9 +2,10 @@ use std::path::PathBuf; use std::time::Duration; use anyhow::bail; +use log::{error, info}; use tokio::fs; use tokio::task::spawn_blocking; -use tokio::time::interval; +use tokio::time::sleep; use crate::helpers::compression; use super::index_actor::IndexActorHandle; @@ -38,11 +39,12 @@ impl SnapshotService { } pub async fn run(self) { - let mut interval = interval(self.snapshot_period); loop { - interval.tick().await; - self.perform_snapshot().await.unwrap(); + sleep(self.snapshot_period).await; + if let Err(e) = self.perform_snapshot().await { + error!("{}", e); + } } } @@ -57,6 +59,11 @@ impl SnapshotService { fs::create_dir_all(&temp_snapshot_path).await?; let uuids = self.uuid_resolver_handle.snapshot(temp_snapshot_path.clone()).await?; + + if uuids.is_empty() { + return Ok(()) + } + for uuid in uuids { self.update_handle.snapshot(uuid, temp_snapshot_path.clone()).await?; } @@ -69,6 +76,8 @@ impl SnapshotService { fs::rename(temp_snapshot_file, &self.snapshot_path).await?; + info!("Created snapshot in {:?}.", self.snapshot_path); + Ok(()) } } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 5cea8e64d..905bc805f 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -16,7 +16,6 @@ use uuid::Uuid; use super::get_arc_ownership_blocking; use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; -use crate::helpers::compression; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 7cd41833c..88b62fe1f 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -10,8 +10,6 @@ use thiserror::Error; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use crate::helpers::compression; - const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB pub type Result = std::result::Result; @@ -330,10 +328,14 @@ impl UuidStore for HeedUuidStore { let uuid = Uuid::from_slice(uuid)?; entries.push(uuid) } - path.push("index_uuids"); - create_dir_all(&path).unwrap(); - path.push("data.mdb"); - env.copy_to_path(path, CompactionOption::Enabled)?; + + // only perform snapshot if there are indexes + if !entries.is_empty() { + path.push("index_uuids"); + create_dir_all(&path).unwrap(); + path.push("data.mdb"); + env.copy_to_path(path, CompactionOption::Enabled)?; + } Ok(entries) }) .await? From 1f51fc8baf5d46736667623c3bf901645be31a35 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 22 Mar 2021 19:59:19 +0100 Subject: [PATCH 217/522] create indexes snapshots concurrently --- meilisearch-http/src/index_controller/snapshot.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index fc7ef7fed..b3a96baa7 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -64,9 +64,12 @@ impl SnapshotService { return Ok(()) } - for uuid in uuids { - self.update_handle.snapshot(uuid, temp_snapshot_path.clone()).await?; - } + let tasks = uuids + .iter() + .map(|&uuid| self.update_handle.snapshot(uuid, temp_snapshot_path.clone())) + .collect::>(); + + futures::future::try_join_all(tasks).await?; let temp_snapshot_file = temp_snapshot_path.with_extension("temp"); From 3cc3637e2d439f4550582e9edf89df242e052890 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Mar 2021 11:00:50 +0100 Subject: [PATCH 218/522] refactor for tests --- .../src/index_controller/index_actor.rs | 657 ------------------ .../src/index_controller/index_actor/actor.rs | 331 +++++++++ .../index_actor/handle_impl.rs | 141 ++++ .../index_controller/index_actor/message.rs | 64 ++ .../src/index_controller/index_actor/mod.rs | 103 +++ .../src/index_controller/index_actor/store.rs | 105 +++ meilisearch-http/src/index_controller/mod.rs | 17 +- .../src/index_controller/snapshot.rs | 20 +- .../src/index_controller/update_actor.rs | 455 ------------ .../index_controller/update_actor/actor.rs | 226 ++++++ .../update_actor/handle_impl.rs | 96 +++ .../index_controller/update_actor/message.rs | 37 + .../src/index_controller/update_actor/mod.rs | 51 ++ .../index_controller/update_actor/store.rs | 112 +++ .../{ => update_actor}/update_store.rs | 0 .../src/index_controller/uuid_resolver.rs | 343 --------- .../index_controller/uuid_resolver/actor.rs | 94 +++ .../uuid_resolver/handle_impl.rs | 78 +++ .../index_controller/uuid_resolver/message.rs | 33 + .../src/index_controller/uuid_resolver/mod.rs | 45 ++ .../index_controller/uuid_resolver/store.rs | 140 ++++ meilisearch-http/tests/common/server.rs | 2 +- 22 files changed, 1676 insertions(+), 1474 deletions(-) delete mode 100644 meilisearch-http/src/index_controller/index_actor.rs create mode 100644 meilisearch-http/src/index_controller/index_actor/actor.rs create mode 100644 meilisearch-http/src/index_controller/index_actor/handle_impl.rs create mode 100644 meilisearch-http/src/index_controller/index_actor/message.rs create mode 100644 meilisearch-http/src/index_controller/index_actor/mod.rs create mode 100644 meilisearch-http/src/index_controller/index_actor/store.rs delete mode 100644 meilisearch-http/src/index_controller/update_actor.rs create mode 100644 meilisearch-http/src/index_controller/update_actor/actor.rs create mode 100644 meilisearch-http/src/index_controller/update_actor/handle_impl.rs create mode 100644 meilisearch-http/src/index_controller/update_actor/message.rs create mode 100644 meilisearch-http/src/index_controller/update_actor/mod.rs create mode 100644 meilisearch-http/src/index_controller/update_actor/store.rs rename meilisearch-http/src/index_controller/{ => update_actor}/update_store.rs (100%) delete mode 100644 meilisearch-http/src/index_controller/uuid_resolver.rs create mode 100644 meilisearch-http/src/index_controller/uuid_resolver/actor.rs create mode 100644 meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs create mode 100644 meilisearch-http/src/index_controller/uuid_resolver/message.rs create mode 100644 meilisearch-http/src/index_controller/uuid_resolver/mod.rs create mode 100644 meilisearch-http/src/index_controller/uuid_resolver/store.rs diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs deleted file mode 100644 index 9c4feef8a..000000000 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ /dev/null @@ -1,657 +0,0 @@ -use std::collections::HashMap; -use std::fs::{create_dir_all, File}; -use std::future::Future; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use async_stream::stream; -use chrono::{DateTime, Utc}; -use futures::pin_mut; -use futures::stream::StreamExt; -use heed::{CompactionOption, EnvOpenOptions}; -use log::debug; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tokio::fs::remove_dir_all; -use tokio::sync::{mpsc, oneshot, RwLock}; -use tokio::task::spawn_blocking; -use uuid::Uuid; - -use super::update_handler::UpdateHandler; -use super::{get_arc_ownership_blocking, IndexSettings}; -use crate::index::UpdateResult as UResult; -use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; -use crate::index_controller::{ - updates::{Failed, Processed, Processing}, - UpdateMeta, -}; -use crate::option::IndexerOpts; - -pub type Result = std::result::Result; -type AsyncMap = Arc>>; -type UpdateResult = std::result::Result, Failed>; - -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct IndexMeta { - created_at: DateTime, - updated_at: DateTime, - primary_key: Option, -} - -impl IndexMeta { - fn new(index: &Index) -> Result { - let txn = index.read_txn()?; - Self::new_txn(index, &txn) - } - - fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { - let created_at = index.created_at(&txn)?; - let updated_at = index.updated_at(&txn)?; - let primary_key = index.primary_key(&txn)?.map(String::from); - Ok(Self { - primary_key, - updated_at, - created_at, - }) - } -} - -enum IndexMsg { - CreateIndex { - uuid: Uuid, - primary_key: Option, - ret: oneshot::Sender>, - }, - Update { - meta: Processing, - data: std::fs::File, - ret: oneshot::Sender>, - }, - Search { - uuid: Uuid, - query: SearchQuery, - ret: oneshot::Sender>, - }, - Settings { - uuid: Uuid, - ret: oneshot::Sender>, - }, - Documents { - uuid: Uuid, - attributes_to_retrieve: Option>, - offset: usize, - limit: usize, - ret: oneshot::Sender>>, - }, - Document { - uuid: Uuid, - attributes_to_retrieve: Option>, - doc_id: String, - ret: oneshot::Sender>, - }, - Delete { - uuid: Uuid, - ret: oneshot::Sender>, - }, - GetMeta { - uuid: Uuid, - ret: oneshot::Sender>, - }, - UpdateIndex { - uuid: Uuid, - index_settings: IndexSettings, - ret: oneshot::Sender>, - }, - Snapshot { - uuid: Uuid, - path: PathBuf, - ret: oneshot::Sender>, - }, -} - -struct IndexActor { - read_receiver: Option>, - write_receiver: Option>, - update_handler: Arc, - store: S, -} - -#[derive(Error, Debug)] -pub enum IndexError { - #[error("error with index: {0}")] - Error(#[from] anyhow::Error), - #[error("index already exists")] - IndexAlreadyExists, - #[error("Index doesn't exists")] - UnexistingIndex, - #[error("Heed error: {0}")] - HeedError(#[from] heed::Error), - #[error("Existing primary key")] - ExistingPrimaryKey, -} - -#[async_trait::async_trait] -trait IndexStore { - async fn create(&self, uuid: Uuid, primary_key: Option) -> Result; - async fn get(&self, uuid: Uuid) -> Result>; - async fn delete(&self, uuid: Uuid) -> Result>; -} - -impl IndexActor { - fn new( - read_receiver: mpsc::Receiver, - write_receiver: mpsc::Receiver, - store: S, - ) -> Result { - let options = IndexerOpts::default(); - let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; - let update_handler = Arc::new(update_handler); - let read_receiver = Some(read_receiver); - let write_receiver = Some(write_receiver); - Ok(Self { - read_receiver, - write_receiver, - store, - update_handler, - }) - } - - /// `run` poll the write_receiver and read_receiver concurrently, but while messages send - /// through the read channel are processed concurrently, the messages sent through the write - /// channel are processed one at a time. - async fn run(mut self) { - let mut read_receiver = self - .read_receiver - .take() - .expect("Index Actor must have a inbox at this point."); - - let read_stream = stream! { - loop { - match read_receiver.recv().await { - Some(msg) => yield msg, - None => break, - } - } - }; - - let mut write_receiver = self - .write_receiver - .take() - .expect("Index Actor must have a inbox at this point."); - - let write_stream = stream! { - loop { - match write_receiver.recv().await { - Some(msg) => yield msg, - None => break, - } - } - }; - - pin_mut!(write_stream); - pin_mut!(read_stream); - - let fut1 = read_stream.for_each_concurrent(Some(10), |msg| self.handle_message(msg)); - let fut2 = write_stream.for_each_concurrent(Some(1), |msg| self.handle_message(msg)); - - let fut1: Box + Unpin + Send> = Box::new(fut1); - let fut2: Box + Unpin + Send> = Box::new(fut2); - - tokio::join!(fut1, fut2); - } - - async fn handle_message(&self, msg: IndexMsg) { - use IndexMsg::*; - match msg { - CreateIndex { - uuid, - primary_key, - ret, - } => { - let _ = ret.send(self.handle_create_index(uuid, primary_key).await); - } - Update { ret, meta, data } => { - let _ = ret.send(self.handle_update(meta, data).await); - } - Search { ret, query, uuid } => { - let _ = ret.send(self.handle_search(uuid, query).await); - } - Settings { ret, uuid } => { - let _ = ret.send(self.handle_settings(uuid).await); - } - Documents { - ret, - uuid, - attributes_to_retrieve, - offset, - limit, - } => { - let _ = ret.send( - self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve) - .await, - ); - } - Document { - uuid, - attributes_to_retrieve, - doc_id, - ret, - } => { - let _ = ret.send( - self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve) - .await, - ); - } - Delete { uuid, ret } => { - let _ = ret.send(self.handle_delete(uuid).await); - } - GetMeta { uuid, ret } => { - let _ = ret.send(self.handle_get_meta(uuid).await); - } - UpdateIndex { - uuid, - index_settings, - ret, - } => { - let _ = ret.send(self.handle_update_index(uuid, index_settings).await); - } - Snapshot { uuid, path, ret } => { - let _ = ret.send(self.handle_snapshot(uuid, path).await); - } - } - } - - async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> anyhow::Result { - let index = self - .store - .get(uuid) - .await? - .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || index.perform_search(query)).await? - } - - async fn handle_create_index( - &self, - uuid: Uuid, - primary_key: Option, - ) -> Result { - let index = self.store.create(uuid, primary_key).await?; - let meta = spawn_blocking(move || IndexMeta::new(&index)) - .await - .map_err(|e| IndexError::Error(e.into()))??; - Ok(meta) - } - - async fn handle_update( - &self, - meta: Processing, - data: File, - ) -> Result { - log::info!("Processing update {}", meta.id()); - let uuid = meta.index_uuid(); - let update_handler = self.update_handler.clone(); - let index = match self.store.get(*uuid).await? { - Some(index) => index, - None => self.store.create(*uuid, None).await?, - }; - spawn_blocking(move || update_handler.handle_update(meta, data, index)) - .await - .map_err(|e| IndexError::Error(e.into())) - } - - async fn handle_settings(&self, uuid: Uuid) -> Result { - let index = self - .store - .get(uuid) - .await? - .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || index.settings().map_err(IndexError::Error)) - .await - .map_err(|e| IndexError::Error(e.into()))? - } - - async fn handle_fetch_documents( - &self, - uuid: Uuid, - offset: usize, - limit: usize, - attributes_to_retrieve: Option>, - ) -> Result> { - let index = self - .store - .get(uuid) - .await? - .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || { - index - .retrieve_documents(offset, limit, attributes_to_retrieve) - .map_err(IndexError::Error) - }) - .await - .map_err(|e| IndexError::Error(e.into()))? - } - - async fn handle_fetch_document( - &self, - uuid: Uuid, - doc_id: String, - attributes_to_retrieve: Option>, - ) -> Result { - let index = self - .store - .get(uuid) - .await? - .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || { - index - .retrieve_document(doc_id, attributes_to_retrieve) - .map_err(IndexError::Error) - }) - .await - .map_err(|e| IndexError::Error(e.into()))? - } - - async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let index = self.store.delete(uuid).await?; - - if let Some(index) = index { - tokio::task::spawn(async move { - let index = index.0; - let store = get_arc_ownership_blocking(index).await; - spawn_blocking(move || { - store.prepare_for_closing().wait(); - debug!("Index closed"); - }); - }); - } - - Ok(()) - } - - async fn handle_get_meta(&self, uuid: Uuid) -> Result { - match self.store.get(uuid).await? { - Some(index) => { - let meta = spawn_blocking(move || IndexMeta::new(&index)) - .await - .map_err(|e| IndexError::Error(e.into()))??; - Ok(meta) - } - None => Err(IndexError::UnexistingIndex), - } - } - - async fn handle_update_index( - &self, - uuid: Uuid, - index_settings: IndexSettings, - ) -> Result { - let index = self - .store - .get(uuid) - .await? - .ok_or(IndexError::UnexistingIndex)?; - - spawn_blocking(move || match index_settings.primary_key { - Some(ref primary_key) => { - let mut txn = index.write_txn()?; - if index.primary_key(&txn)?.is_some() { - return Err(IndexError::ExistingPrimaryKey); - } - index.put_primary_key(&mut txn, primary_key)?; - let meta = IndexMeta::new_txn(&index, &txn)?; - txn.commit()?; - Ok(meta) - } - None => { - let meta = IndexMeta::new(&index)?; - Ok(meta) - } - }) - .await - .map_err(|e| IndexError::Error(e.into()))? - } - - async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> { - use tokio::fs::create_dir_all; - - path.push("indexes"); - create_dir_all(&path) - .await - .map_err(|e| IndexError::Error(e.into()))?; - - if let Some(index) = self.store.get(uuid).await? { - let mut index_path = path.join(format!("index-{}", uuid)); - create_dir_all(&index_path) - .await - .map_err(|e| IndexError::Error(e.into()))?; - index_path.push("data.mdb"); - spawn_blocking(move || -> anyhow::Result<()> { - // Get write txn to wait for ongoing write transaction before snapshot. - let _txn = index.write_txn()?; - index - .env - .copy_to_path(index_path, CompactionOption::Enabled)?; - Ok(()) - }) - .await - .map_err(|e| IndexError::Error(e.into()))? - .map_err(|e| IndexError::Error(e.into()))?; - } - - Ok(()) - } -} - -#[derive(Clone)] -pub struct IndexActorHandle { - read_sender: mpsc::Sender, - write_sender: mpsc::Sender, -} - -impl IndexActorHandle { - pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { - let (read_sender, read_receiver) = mpsc::channel(100); - let (write_sender, write_receiver) = mpsc::channel(100); - - let store = HeedIndexStore::new(path, index_size); - let actor = IndexActor::new(read_receiver, write_receiver, store)?; - tokio::task::spawn(actor.run()); - Ok(Self { - read_sender, - write_sender, - }) - } - - pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::CreateIndex { - ret, - uuid, - primary_key, - }; - let _ = self.read_sender.send(msg).await; - receiver.await.expect("IndexActor has been killed") - } - - pub async fn update( - &self, - meta: Processing, - data: std::fs::File, - ) -> anyhow::Result { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Update { ret, meta, data }; - let _ = self.write_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } - - pub async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Search { uuid, query, ret }; - let _ = self.read_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } - - pub async fn settings(&self, uuid: Uuid) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Settings { uuid, ret }; - let _ = self.read_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } - - pub async fn documents( - &self, - uuid: Uuid, - offset: usize, - limit: usize, - attributes_to_retrieve: Option>, - ) -> Result> { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Documents { - uuid, - ret, - offset, - attributes_to_retrieve, - limit, - }; - let _ = self.read_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } - - pub async fn document( - &self, - uuid: Uuid, - doc_id: String, - attributes_to_retrieve: Option>, - ) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Document { - uuid, - ret, - doc_id, - attributes_to_retrieve, - }; - let _ = self.read_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } - - pub async fn delete(&self, uuid: Uuid) -> Result<()> { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Delete { uuid, ret }; - let _ = self.write_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } - - pub async fn get_index_meta(&self, uuid: Uuid) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::GetMeta { uuid, ret }; - let _ = self.read_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } - - pub async fn update_index( - &self, - uuid: Uuid, - index_settings: IndexSettings, - ) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::UpdateIndex { - uuid, - index_settings, - ret, - }; - let _ = self.read_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } - - pub async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { - let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Snapshot { uuid, path, ret }; - let _ = self.read_sender.send(msg).await; - Ok(receiver.await.expect("IndexActor has been killed")?) - } -} - -struct HeedIndexStore { - index_store: AsyncMap, - path: PathBuf, - index_size: usize, -} - -impl HeedIndexStore { - fn new(path: impl AsRef, index_size: usize) -> Self { - let path = path.as_ref().join("indexes/"); - let index_store = Arc::new(RwLock::new(HashMap::new())); - Self { - index_store, - path, - index_size, - } - } -} - -#[async_trait::async_trait] -impl IndexStore for HeedIndexStore { - async fn create(&self, uuid: Uuid, primary_key: Option) -> Result { - let path = self.path.join(format!("index-{}", uuid)); - if path.exists() { - return Err(IndexError::IndexAlreadyExists); - } - - let index_size = self.index_size; - let index = spawn_blocking(move || -> Result { - let index = open_index(&path, index_size)?; - if let Some(primary_key) = primary_key { - let mut txn = index.write_txn()?; - index.put_primary_key(&mut txn, &primary_key)?; - txn.commit()?; - } - Ok(index) - }) - .await - .map_err(|e| IndexError::Error(e.into()))??; - - self.index_store.write().await.insert(uuid, index.clone()); - - Ok(index) - } - - async fn get(&self, uuid: Uuid) -> Result> { - let guard = self.index_store.read().await; - match guard.get(&uuid) { - Some(index) => Ok(Some(index.clone())), - None => { - // drop the guard here so we can perform the write after without deadlocking; - drop(guard); - let path = self.path.join(format!("index-{}", uuid)); - if !path.exists() { - return Ok(None); - } - - let index_size = self.index_size; - let index = spawn_blocking(move || open_index(path, index_size)) - .await - .map_err(|e| IndexError::Error(e.into()))??; - self.index_store.write().await.insert(uuid, index.clone()); - Ok(Some(index)) - } - } - } - - async fn delete(&self, uuid: Uuid) -> Result> { - let db_path = self.path.join(format!("index-{}", uuid)); - remove_dir_all(db_path) - .await - .map_err(|e| IndexError::Error(e.into()))?; - let index = self.index_store.write().await.remove(&uuid); - Ok(index) - } -} - -fn open_index(path: impl AsRef, size: usize) -> Result { - create_dir_all(&path).map_err(|e| IndexError::Error(e.into()))?; - let mut options = EnvOpenOptions::new(); - options.map_size(size); - let index = milli::Index::new(options, &path).map_err(IndexError::Error)?; - Ok(Index(Arc::new(index))) -} diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs new file mode 100644 index 000000000..89831b53a --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -0,0 +1,331 @@ +use std::fs::File; +use std::future::Future; +use std::path::PathBuf; +use std::sync::Arc; + +use async_stream::stream; +use futures::pin_mut; +use futures::stream::StreamExt; +use heed::CompactionOption; +use log::debug; +use tokio::sync::mpsc; +use tokio::task::spawn_blocking; +use uuid::Uuid; + +use crate::index::{Document, SearchQuery, SearchResult, Settings}; +use crate::index_controller::update_handler::UpdateHandler; +use crate::index_controller::{updates::Processing, UpdateMeta, get_arc_ownership_blocking}; +use crate::option::IndexerOpts; +use super::{IndexSettings, Result, IndexMsg, IndexStore, IndexError, UpdateResult, IndexMeta}; + +pub struct IndexActor { + read_receiver: Option>, + write_receiver: Option>, + update_handler: Arc, + store: S, +} + +impl IndexActor { + pub fn new( + read_receiver: mpsc::Receiver, + write_receiver: mpsc::Receiver, + store: S, + ) -> Result { + let options = IndexerOpts::default(); + let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; + let update_handler = Arc::new(update_handler); + let read_receiver = Some(read_receiver); + let write_receiver = Some(write_receiver); + Ok(Self { + read_receiver, + write_receiver, + store, + update_handler, + }) + } + + /// `run` poll the write_receiver and read_receiver concurrently, but while messages send + /// through the read channel are processed concurrently, the messages sent through the write + /// channel are processed one at a time. + pub async fn run(mut self) { + let mut read_receiver = self + .read_receiver + .take() + .expect("Index Actor must have a inbox at this point."); + + let read_stream = stream! { + loop { + match read_receiver.recv().await { + Some(msg) => yield msg, + None => break, + } + } + }; + + let mut write_receiver = self + .write_receiver + .take() + .expect("Index Actor must have a inbox at this point."); + + let write_stream = stream! { + loop { + match write_receiver.recv().await { + Some(msg) => yield msg, + None => break, + } + } + }; + + pin_mut!(write_stream); + pin_mut!(read_stream); + + let fut1 = read_stream.for_each_concurrent(Some(10), |msg| self.handle_message(msg)); + let fut2 = write_stream.for_each_concurrent(Some(1), |msg| self.handle_message(msg)); + + let fut1: Box + Unpin + Send> = Box::new(fut1); + let fut2: Box + Unpin + Send> = Box::new(fut2); + + tokio::join!(fut1, fut2); + } + + async fn handle_message(&self, msg: IndexMsg) { + use IndexMsg::*; + match msg { + CreateIndex { + uuid, + primary_key, + ret, + } => { + let _ = ret.send(self.handle_create_index(uuid, primary_key).await); + } + Update { ret, meta, data } => { + let _ = ret.send(self.handle_update(meta, data).await); + } + Search { ret, query, uuid } => { + let _ = ret.send(self.handle_search(uuid, query).await); + } + Settings { ret, uuid } => { + let _ = ret.send(self.handle_settings(uuid).await); + } + Documents { + ret, + uuid, + attributes_to_retrieve, + offset, + limit, + } => { + let _ = ret.send( + self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve) + .await, + ); + } + Document { + uuid, + attributes_to_retrieve, + doc_id, + ret, + } => { + let _ = ret.send( + self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve) + .await, + ); + } + Delete { uuid, ret } => { + let _ = ret.send(self.handle_delete(uuid).await); + } + GetMeta { uuid, ret } => { + let _ = ret.send(self.handle_get_meta(uuid).await); + } + UpdateIndex { + uuid, + index_settings, + ret, + } => { + let _ = ret.send(self.handle_update_index(uuid, index_settings).await); + } + Snapshot { uuid, path, ret } => { + let _ = ret.send(self.handle_snapshot(uuid, path).await); + } + } + } + + async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> anyhow::Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + spawn_blocking(move || index.perform_search(query)).await? + } + + async fn handle_create_index( + &self, + uuid: Uuid, + primary_key: Option, + ) -> Result { + let index = self.store.create(uuid, primary_key).await?; + let meta = spawn_blocking(move || IndexMeta::new(&index)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + Ok(meta) + } + + async fn handle_update( + &self, + meta: Processing, + data: File, + ) -> Result { + debug!("Processing update {}", meta.id()); + let uuid = meta.index_uuid(); + let update_handler = self.update_handler.clone(); + let index = match self.store.get(*uuid).await? { + Some(index) => index, + None => self.store.create(*uuid, None).await?, + }; + spawn_blocking(move || update_handler.handle_update(meta, data, index)) + .await + .map_err(|e| IndexError::Error(e.into())) + } + + async fn handle_settings(&self, uuid: Uuid) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + spawn_blocking(move || index.settings().map_err(IndexError::Error)) + .await + .map_err(|e| IndexError::Error(e.into()))? + } + + async fn handle_fetch_documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + spawn_blocking(move || { + index + .retrieve_documents(offset, limit, attributes_to_retrieve) + .map_err(IndexError::Error) + }) + .await + .map_err(|e| IndexError::Error(e.into()))? + } + + async fn handle_fetch_document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + spawn_blocking(move || { + index + .retrieve_document(doc_id, attributes_to_retrieve) + .map_err(IndexError::Error) + }) + .await + .map_err(|e| IndexError::Error(e.into()))? + } + + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + let index = self.store.delete(uuid).await?; + + if let Some(index) = index { + tokio::task::spawn(async move { + let index = index.0; + let store = get_arc_ownership_blocking(index).await; + spawn_blocking(move || { + store.prepare_for_closing().wait(); + debug!("Index closed"); + }); + }); + } + + Ok(()) + } + + async fn handle_get_meta(&self, uuid: Uuid) -> Result { + match self.store.get(uuid).await? { + Some(index) => { + let meta = spawn_blocking(move || IndexMeta::new(&index)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + Ok(meta) + } + None => Err(IndexError::UnexistingIndex), + } + } + + async fn handle_update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + + spawn_blocking(move || match index_settings.primary_key { + Some(ref primary_key) => { + let mut txn = index.write_txn()?; + if index.primary_key(&txn)?.is_some() { + return Err(IndexError::ExistingPrimaryKey); + } + index.put_primary_key(&mut txn, primary_key)?; + let meta = IndexMeta::new_txn(&index, &txn)?; + txn.commit()?; + Ok(meta) + } + None => { + let meta = IndexMeta::new(&index)?; + Ok(meta) + } + }) + .await + .map_err(|e| IndexError::Error(e.into()))? + } + + async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> { + use tokio::fs::create_dir_all; + + path.push("indexes"); + create_dir_all(&path) + .await + .map_err(|e| IndexError::Error(e.into()))?; + + if let Some(index) = self.store.get(uuid).await? { + let mut index_path = path.join(format!("index-{}", uuid)); + create_dir_all(&index_path) + .await + .map_err(|e| IndexError::Error(e.into()))?; + index_path.push("data.mdb"); + spawn_blocking(move || -> anyhow::Result<()> { + // Get write txn to wait for ongoing write transaction before snapshot. + let _txn = index.write_txn()?; + index + .env + .copy_to_path(index_path, CompactionOption::Enabled)?; + Ok(()) + }) + .await + .map_err(|e| IndexError::Error(e.into()))? + .map_err(|e| IndexError::Error(e.into()))?; + } + + Ok(()) + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs new file mode 100644 index 000000000..9c43bd6e7 --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -0,0 +1,141 @@ +use std::path::{PathBuf, Path}; + +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; + +use crate::index::{Document, SearchQuery, SearchResult, Settings}; +use crate::index_controller::IndexSettings; +use crate::index_controller::{updates::Processing, UpdateMeta}; +use super::{IndexActorHandle, IndexMsg, IndexMeta, UpdateResult, Result, IndexActor, MapIndexStore}; + +#[derive(Clone)] +pub struct IndexActorHandleImpl { + read_sender: mpsc::Sender, + write_sender: mpsc::Sender, +} + +#[async_trait::async_trait] +impl IndexActorHandle for IndexActorHandleImpl { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::CreateIndex { + ret, + uuid, + primary_key, + }; + let _ = self.read_sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } + + async fn update( + &self, + meta: Processing, + data: std::fs::File, + ) -> anyhow::Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Update { ret, meta, data }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Search { uuid, query, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn settings(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Settings { uuid, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Documents { + uuid, + ret, + offset, + attributes_to_retrieve, + limit, + }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Document { + uuid, + ret, + doc_id, + attributes_to_retrieve, + }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn delete(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Delete { uuid, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn get_index_meta(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::GetMeta { uuid, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::UpdateIndex { + uuid, + index_settings, + ret, + }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Snapshot { uuid, path, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } +} + +impl IndexActorHandleImpl { + pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { + let (read_sender, read_receiver) = mpsc::channel(100); + let (write_sender, write_receiver) = mpsc::channel(100); + + let store = MapIndexStore::new(path, index_size); + let actor = IndexActor::new(read_receiver, write_receiver, store)?; + tokio::task::spawn(actor.run()); + Ok(Self { + read_sender, + write_sender, + }) + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs new file mode 100644 index 000000000..66edb5b77 --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -0,0 +1,64 @@ +use std::path::PathBuf; + +use tokio::sync::oneshot; +use uuid::Uuid; + +use crate::index::{Document, SearchQuery, SearchResult, Settings}; +use crate::index_controller::{ + updates::Processing, + UpdateMeta, +}; +use super::{IndexSettings, IndexMeta, UpdateResult, Result}; + +pub enum IndexMsg { + CreateIndex { + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + }, + Update { + meta: Processing, + data: std::fs::File, + ret: oneshot::Sender>, + }, + Search { + uuid: Uuid, + query: SearchQuery, + ret: oneshot::Sender>, + }, + Settings { + uuid: Uuid, + ret: oneshot::Sender>, + }, + Documents { + uuid: Uuid, + attributes_to_retrieve: Option>, + offset: usize, + limit: usize, + ret: oneshot::Sender>>, + }, + Document { + uuid: Uuid, + attributes_to_retrieve: Option>, + doc_id: String, + ret: oneshot::Sender>, + }, + Delete { + uuid: Uuid, + ret: oneshot::Sender>, + }, + GetMeta { + uuid: Uuid, + ret: oneshot::Sender>, + }, + UpdateIndex { + uuid: Uuid, + index_settings: IndexSettings, + ret: oneshot::Sender>, + }, + Snapshot { + uuid: Uuid, + path: PathBuf, + ret: oneshot::Sender>, + }, +} diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs new file mode 100644 index 000000000..dac7ef06c --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -0,0 +1,103 @@ +mod actor; +mod handle_impl; +mod message; +mod store; + +use std::path::PathBuf; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +use super::IndexSettings; +use crate::index::UpdateResult as UResult; +use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; +use crate::index_controller::{ + updates::{Failed, Processed, Processing}, + UpdateMeta, +}; +use message::IndexMsg; +use store::{IndexStore, MapIndexStore}; +use actor::IndexActor; + +pub use handle_impl::IndexActorHandleImpl; + +pub type Result = std::result::Result; +type UpdateResult = std::result::Result, Failed>; + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IndexMeta { + created_at: DateTime, + updated_at: DateTime, + primary_key: Option, +} + +impl IndexMeta { + fn new(index: &Index) -> Result { + let txn = index.read_txn()?; + Self::new_txn(index, &txn) + } + + fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { + let created_at = index.created_at(&txn)?; + let updated_at = index.updated_at(&txn)?; + let primary_key = index.primary_key(&txn)?.map(String::from); + Ok(Self { + primary_key, + updated_at, + created_at, + }) + } +} + +#[derive(Error, Debug)] +pub enum IndexError { + #[error("error with index: {0}")] + Error(#[from] anyhow::Error), + #[error("index already exists")] + IndexAlreadyExists, + #[error("Index doesn't exists")] + UnexistingIndex, + #[error("Heed error: {0}")] + HeedError(#[from] heed::Error), + #[error("Existing primary key")] + ExistingPrimaryKey, +} + + +#[async_trait::async_trait] +pub trait IndexActorHandle: Sync + Send + Clone { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn update( + &self, + meta: Processing, + data: std::fs::File, + ) -> anyhow::Result; + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result; + async fn settings(&self, uuid: Uuid) -> Result; + + async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result>; + async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result; + async fn delete(&self, uuid: Uuid) -> Result<()>; + async fn get_index_meta(&self, uuid: Uuid) -> Result; + async fn update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> Result; + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; +} + diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs new file mode 100644 index 000000000..a9f3cd479 --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -0,0 +1,105 @@ +use std::path::{PathBuf, Path}; +use std::sync::Arc; +use std::collections::HashMap; + +use uuid::Uuid; +use tokio::sync::RwLock; +use tokio::task::spawn_blocking; +use tokio::fs; +use heed::EnvOpenOptions; + +use super::{IndexError, Result}; +use crate::index::Index; + +type AsyncMap = Arc>>; + +#[async_trait::async_trait] +pub trait IndexStore { + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn get(&self, uuid: Uuid) -> Result>; + async fn delete(&self, uuid: Uuid) -> Result>; +} + +pub struct MapIndexStore { + index_store: AsyncMap, + path: PathBuf, + index_size: usize, +} + +impl MapIndexStore { + pub fn new(path: impl AsRef, index_size: usize) -> Self { + let path = path.as_ref().join("indexes/"); + let index_store = Arc::new(RwLock::new(HashMap::new())); + Self { + index_store, + path, + index_size, + } + } +} + +#[async_trait::async_trait] +impl IndexStore for MapIndexStore { + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result { + let path = self.path.join(format!("index-{}", uuid)); + if path.exists() { + return Err(IndexError::IndexAlreadyExists); + } + + let index_size = self.index_size; + let index = spawn_blocking(move || -> Result { + let index = open_index(&path, index_size)?; + if let Some(primary_key) = primary_key { + let mut txn = index.write_txn()?; + index.put_primary_key(&mut txn, &primary_key)?; + txn.commit()?; + } + Ok(index) + }) + .await + .map_err(|e| IndexError::Error(e.into()))??; + + self.index_store.write().await.insert(uuid, index.clone()); + + Ok(index) + } + + async fn get(&self, uuid: Uuid) -> Result> { + let guard = self.index_store.read().await; + match guard.get(&uuid) { + Some(index) => Ok(Some(index.clone())), + None => { + // drop the guard here so we can perform the write after without deadlocking; + drop(guard); + let path = self.path.join(format!("index-{}", uuid)); + if !path.exists() { + return Ok(None); + } + + let index_size = self.index_size; + let index = spawn_blocking(move || open_index(path, index_size)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + self.index_store.write().await.insert(uuid, index.clone()); + Ok(Some(index)) + } + } + } + + async fn delete(&self, uuid: Uuid) -> Result> { + let db_path = self.path.join(format!("index-{}", uuid)); + fs::remove_dir_all(db_path) + .await + .map_err(|e| IndexError::Error(e.into()))?; + let index = self.index_store.write().await.remove(&uuid); + Ok(index) + } +} + +fn open_index(path: impl AsRef, size: usize) -> Result { + std::fs::create_dir_all(&path).map_err(|e| IndexError::Error(e.into()))?; + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, &path).map_err(IndexError::Error)?; + Ok(Index(Arc::new(index))) +} diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 9fab9088a..bcd0cd8fe 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -2,7 +2,6 @@ mod index_actor; mod snapshot; mod update_actor; mod update_handler; -mod update_store; mod updates; mod uuid_resolver; @@ -22,6 +21,9 @@ use crate::index::{Document, SearchQuery, SearchResult}; use crate::index::{Facets, Settings, UpdateResult}; use crate::option::Opt; use crate::helpers::compression; +use index_actor::IndexActorHandle; +use update_actor::UpdateActorHandle; +use uuid_resolver::UuidResolverHandle; use snapshot::SnapshotService; pub use updates::{Failed, Processed, Processing}; @@ -58,9 +60,9 @@ pub struct IndexSettings { } pub struct IndexController { - uuid_resolver: uuid_resolver::UuidResolverHandle, - index_handle: index_actor::IndexActorHandle, - update_handle: update_actor::UpdateActorHandle, + uuid_resolver: uuid_resolver::UuidResolverHandleImpl, + index_handle: index_actor::IndexActorHandleImpl, + update_handle: update_actor::UpdateActorHandleImpl, } impl IndexController { @@ -72,9 +74,9 @@ impl IndexController { compression::from_tar_gz(path, &options.db_path)?; } - let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; - let index_handle = index_actor::IndexActorHandle::new(&path, index_size)?; - let update_handle = update_actor::UpdateActorHandle::new( + let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&path)?; + let index_handle = index_actor::IndexActorHandleImpl::new(&path, index_size)?; + let update_handle = update_actor::UpdateActorHandleImpl::new( index_handle.clone(), &path, update_store_size, @@ -82,7 +84,6 @@ impl IndexController { if options.schedule_snapshot { let snapshot_service = SnapshotService::new( - index_handle.clone(), uuid_resolver.clone(), update_handle.clone(), Duration::from_secs(options.snapshot_interval_sec), diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index b3a96baa7..afdcdaf23 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -8,29 +8,29 @@ use tokio::task::spawn_blocking; use tokio::time::sleep; use crate::helpers::compression; -use super::index_actor::IndexActorHandle; use super::update_actor::UpdateActorHandle; use super::uuid_resolver::UuidResolverHandle; #[allow(dead_code)] -pub struct SnapshotService { - index_handle: IndexActorHandle, - uuid_resolver_handle: UuidResolverHandle, - update_handle: UpdateActorHandle, +pub struct SnapshotService { + uuid_resolver_handle: R, + update_handle: U, snapshot_period: Duration, snapshot_path: PathBuf, } -impl SnapshotService { +impl SnapshotService +where + U: UpdateActorHandle, + R: UuidResolverHandle +{ pub fn new( - index_handle: IndexActorHandle, - uuid_resolver_handle: UuidResolverHandle, - update_handle: UpdateActorHandle, + uuid_resolver_handle: R, + update_handle: U, snapshot_period: Duration, snapshot_path: PathBuf, ) -> Self { Self { - index_handle, uuid_resolver_handle, update_handle, snapshot_period, diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs deleted file mode 100644 index 905bc805f..000000000 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ /dev/null @@ -1,455 +0,0 @@ -use std::collections::{hash_map::Entry, HashMap}; -use std::fs; -use std::io::SeekFrom; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use super::index_actor::IndexActorHandle; -use log::info; -use oxidized_json_checker::JsonChecker; -use thiserror::Error; -use tokio::fs::OpenOptions; -use tokio::io::{AsyncSeekExt, AsyncWriteExt}; -use tokio::sync::{mpsc, oneshot, RwLock}; -use uuid::Uuid; - -use super::get_arc_ownership_blocking; -use crate::index::UpdateResult; -use crate::index_controller::{UpdateMeta, UpdateStatus}; - -pub type Result = std::result::Result; -type UpdateStore = super::update_store::UpdateStore; -type PayloadData = std::result::Result>; - -#[derive(Debug, Error)] -pub enum UpdateError { - #[error("error with update: {0}")] - Error(Box), - #[error("Index {0} doesn't exist.")] - UnexistingIndex(Uuid), - #[error("Update {0} doesn't exist.")] - UnexistingUpdate(u64), -} - -enum UpdateMsg { - Update { - uuid: Uuid, - meta: UpdateMeta, - data: mpsc::Receiver>, - ret: oneshot::Sender>, - }, - ListUpdates { - uuid: Uuid, - ret: oneshot::Sender>>, - }, - GetUpdate { - uuid: Uuid, - ret: oneshot::Sender>, - id: u64, - }, - Delete { - uuid: Uuid, - ret: oneshot::Sender>, - }, - Create { - uuid: Uuid, - ret: oneshot::Sender>, - }, - Snapshot { - uuid: Uuid, - path: PathBuf, - ret: oneshot::Sender>, - }, -} - -struct UpdateActor { - path: PathBuf, - store: S, - inbox: mpsc::Receiver>, - index_handle: IndexActorHandle, -} - -#[async_trait::async_trait] -trait UpdateStoreStore { - async fn get_or_create(&self, uuid: Uuid) -> Result>; - async fn delete(&self, uuid: Uuid) -> Result>>; - async fn get(&self, uuid: Uuid) -> Result>>; -} - -impl UpdateActor -where - D: AsRef<[u8]> + Sized + 'static, - S: UpdateStoreStore, -{ - fn new( - store: S, - inbox: mpsc::Receiver>, - path: impl AsRef, - index_handle: IndexActorHandle, - ) -> anyhow::Result { - let path = path.as_ref().to_owned(); - fs::create_dir_all(path.join("update_files"))?; - assert!(path.exists()); - Ok(Self { - store, - inbox, - path, - index_handle, - }) - } - - async fn run(mut self) { - use UpdateMsg::*; - - info!("Started update actor."); - - loop { - match self.inbox.recv().await { - Some(Update { - uuid, - meta, - data, - ret, - }) => { - let _ = ret.send(self.handle_update(uuid, meta, data).await); - } - Some(ListUpdates { uuid, ret }) => { - let _ = ret.send(self.handle_list_updates(uuid).await); - } - Some(GetUpdate { uuid, ret, id }) => { - let _ = ret.send(self.handle_get_update(uuid, id).await); - } - Some(Delete { uuid, ret }) => { - let _ = ret.send(self.handle_delete(uuid).await); - } - Some(Create { uuid, ret }) => { - let _ = ret.send(self.handle_create(uuid).await); - } - Some(Snapshot { uuid, path, ret }) => { - let _ = ret.send(self.handle_snapshot(uuid, path).await); - } - None => break, - } - } - } - - async fn handle_update( - &self, - uuid: Uuid, - meta: UpdateMeta, - mut payload: mpsc::Receiver>, - ) -> Result { - let update_store = self.store.get_or_create(uuid).await?; - let update_file_id = uuid::Uuid::new_v4(); - let path = self - .path - .join(format!("update_files/update_{}", update_file_id)); - let mut file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .open(&path) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; - - while let Some(bytes) = payload.recv().await { - match bytes { - Ok(bytes) => { - file.write_all(bytes.as_ref()) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; - } - Err(e) => { - return Err(UpdateError::Error(e)); - } - } - } - - file.flush() - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; - - file.seek(SeekFrom::Start(0)) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; - - let mut file = file.into_std().await; - - tokio::task::spawn_blocking(move || { - use std::io::{copy, sink, BufReader, Seek}; - - // If the payload is empty, ignore the check. - if file - .metadata() - .map_err(|e| UpdateError::Error(Box::new(e)))? - .len() - > 0 - { - // Check that the json payload is valid: - let reader = BufReader::new(&mut file); - let mut checker = JsonChecker::new(reader); - - if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() { - // The json file is invalid, we use Serde to get a nice error message: - file.seek(SeekFrom::Start(0)) - .map_err(|e| UpdateError::Error(Box::new(e)))?; - let _: serde_json::Value = serde_json::from_reader(file) - .map_err(|e| UpdateError::Error(Box::new(e)))?; - } - } - - // The payload is valid, we can register it to the update store. - update_store - .register_update(meta, path, uuid) - .map(UpdateStatus::Pending) - .map_err(|e| UpdateError::Error(Box::new(e))) - }) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))? - } - - async fn handle_list_updates(&self, uuid: Uuid) -> Result> { - let update_store = self.store.get(uuid).await?; - tokio::task::spawn_blocking(move || { - let result = update_store - .ok_or(UpdateError::UnexistingIndex(uuid))? - .list() - .map_err(|e| UpdateError::Error(e.into()))?; - Ok(result) - }) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))? - } - - async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { - let store = self - .store - .get(uuid) - .await? - .ok_or(UpdateError::UnexistingIndex(uuid))?; - let result = store - .meta(id) - .map_err(|e| UpdateError::Error(Box::new(e)))? - .ok_or(UpdateError::UnexistingUpdate(id))?; - Ok(result) - } - - async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let store = self.store.delete(uuid).await?; - - if let Some(store) = store { - tokio::task::spawn(async move { - let store = get_arc_ownership_blocking(store).await; - tokio::task::spawn_blocking(move || { - store.prepare_for_closing().wait(); - info!("Update store {} was closed.", uuid); - }); - }); - } - - Ok(()) - } - - async fn handle_create(&self, uuid: Uuid) -> Result<()> { - let _ = self.store.get_or_create(uuid).await?; - Ok(()) - } - - async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { - let index_handle = self.index_handle.clone(); - if let Some(update_store) = self.store.get(uuid).await? { - tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - // acquire write lock to prevent further writes during snapshot - // the update lock must be acquired BEFORE the write lock to prevent dead lock - let _lock = update_store.update_lock.lock(); - let mut txn = update_store.env.write_txn()?; - - // create db snapshot - update_store.snapshot(&mut txn, &path, uuid)?; - - futures::executor::block_on( - async move { index_handle.snapshot(uuid, path).await }, - )?; - Ok(()) - }) - .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?; - } - - Ok(()) - } -} - -#[derive(Clone)] -pub struct UpdateActorHandle { - sender: mpsc::Sender>, -} - -impl UpdateActorHandle -where - D: AsRef<[u8]> + Sized + 'static + Sync + Send, -{ - pub fn new( - index_handle: IndexActorHandle, - path: impl AsRef, - update_store_size: usize, - ) -> anyhow::Result { - let path = path.as_ref().to_owned().join("updates"); - let (sender, receiver) = mpsc::channel(100); - let store = MapUpdateStoreStore::new(index_handle.clone(), &path, update_store_size); - let actor = UpdateActor::new(store, receiver, path, index_handle)?; - - tokio::task::spawn(actor.run()); - - Ok(Self { sender }) - } - - pub async fn update( - &self, - meta: UpdateMeta, - data: mpsc::Receiver>, - uuid: Uuid, - ) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Update { - uuid, - data, - meta, - ret, - }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } -} - -impl UpdateActorHandle { - pub async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::ListUpdates { uuid, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } - - pub async fn update_status(&self, uuid: Uuid, id: u64) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::GetUpdate { uuid, id, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } - - pub async fn delete(&self, uuid: Uuid) -> Result<()> { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Delete { uuid, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } - - pub async fn create(&self, uuid: Uuid) -> Result<()> { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Create { uuid, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } - - pub async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Snapshot { uuid, path, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } -} - -struct MapUpdateStoreStore { - db: Arc>>>, - index_handle: IndexActorHandle, - path: PathBuf, - update_store_size: usize, -} - -impl MapUpdateStoreStore { - fn new( - index_handle: IndexActorHandle, - path: impl AsRef, - update_store_size: usize, - ) -> Self { - let db = Arc::new(RwLock::new(HashMap::new())); - let path = path.as_ref().to_owned(); - Self { - db, - index_handle, - path, - update_store_size, - } - } -} - -#[async_trait::async_trait] -impl UpdateStoreStore for MapUpdateStoreStore { - async fn get_or_create(&self, uuid: Uuid) -> Result> { - match self.db.write().await.entry(uuid) { - Entry::Vacant(e) => { - let mut options = heed::EnvOpenOptions::new(); - let update_store_size = self.update_store_size; - options.map_size(update_store_size); - let path = self.path.clone().join(format!("updates-{}", e.key())); - fs::create_dir_all(&path).unwrap(); - let index_handle = self.index_handle.clone(); - let store = UpdateStore::open(options, &path, move |meta, file| { - futures::executor::block_on(index_handle.update(meta, file)) - }) - .map_err(|e| UpdateError::Error(e.into()))?; - let store = e.insert(store); - Ok(store.clone()) - } - Entry::Occupied(e) => Ok(e.get().clone()), - } - } - - async fn get(&self, uuid: Uuid) -> Result>> { - let guard = self.db.read().await; - match guard.get(&uuid) { - Some(uuid) => Ok(Some(uuid.clone())), - None => { - // The index is not found in the found in the loaded indexes, so we attempt to load - // it from disk. We need to acquire a write lock **before** attempting to open the - // index, because someone could be trying to open it at the same time as us. - drop(guard); - let path = self.path.clone().join(format!("updates-{}", uuid)); - if path.exists() { - let mut guard = self.db.write().await; - match guard.entry(uuid) { - Entry::Vacant(entry) => { - // We can safely load the index - let index_handle = self.index_handle.clone(); - let mut options = heed::EnvOpenOptions::new(); - let update_store_size = self.update_store_size; - options.map_size(update_store_size); - let store = UpdateStore::open(options, &path, move |meta, file| { - futures::executor::block_on(index_handle.update(meta, file)) - }) - .map_err(|e| UpdateError::Error(e.into()))?; - let store = entry.insert(store); - Ok(Some(store.clone())) - } - Entry::Occupied(entry) => { - // The index was loaded while we attempted to to iter - Ok(Some(entry.get().clone())) - } - } - } else { - Ok(None) - } - } - } - } - - async fn delete(&self, uuid: Uuid) -> Result>> { - let store = self.db.write().await.remove(&uuid); - let path = self.path.clone().join(format!("updates-{}", uuid)); - if store.is_some() || path.exists() { - fs::remove_dir_all(path).unwrap(); - } - Ok(store) - } -} diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs new file mode 100644 index 000000000..629a87ba4 --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -0,0 +1,226 @@ +use std::io::SeekFrom; +use std::path::{Path, PathBuf}; + +use log::info; +use tokio::sync::mpsc; +use uuid::Uuid; +use oxidized_json_checker::JsonChecker; +use tokio::fs; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; + +use super::{PayloadData, UpdateError, UpdateMsg, UpdateStoreStore, Result}; +use crate::index_controller::index_actor::IndexActorHandle; +use crate::index_controller::{UpdateMeta, UpdateStatus, get_arc_ownership_blocking}; + +pub struct UpdateActor { + path: PathBuf, + store: S, + inbox: mpsc::Receiver>, + index_handle: I, +} + +impl UpdateActor +where + D: AsRef<[u8]> + Sized + 'static, + S: UpdateStoreStore, + I: IndexActorHandle + 'static, +{ + pub fn new( + store: S, + inbox: mpsc::Receiver>, + path: impl AsRef, + index_handle: I, + ) -> anyhow::Result { + let path = path.as_ref().to_owned(); + std::fs::create_dir_all(path.join("update_files"))?; + assert!(path.exists()); + Ok(Self { + store, + inbox, + path, + index_handle, + }) + } + + pub async fn run(mut self) { + use UpdateMsg::*; + + info!("Started update actor."); + + loop { + match self.inbox.recv().await { + Some(Update { + uuid, + meta, + data, + ret, + }) => { + let _ = ret.send(self.handle_update(uuid, meta, data).await); + } + Some(ListUpdates { uuid, ret }) => { + let _ = ret.send(self.handle_list_updates(uuid).await); + } + Some(GetUpdate { uuid, ret, id }) => { + let _ = ret.send(self.handle_get_update(uuid, id).await); + } + Some(Delete { uuid, ret }) => { + let _ = ret.send(self.handle_delete(uuid).await); + } + Some(Create { uuid, ret }) => { + let _ = ret.send(self.handle_create(uuid).await); + } + Some(Snapshot { uuid, path, ret }) => { + let _ = ret.send(self.handle_snapshot(uuid, path).await); + } + None => break, + } + } + } + + async fn handle_update( + &self, + uuid: Uuid, + meta: UpdateMeta, + mut payload: mpsc::Receiver>, + ) -> Result { + let update_store = self.store.get_or_create(uuid).await?; + let update_file_id = uuid::Uuid::new_v4(); + let path = self + .path + .join(format!("update_files/update_{}", update_file_id)); + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&path) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))?; + + while let Some(bytes) = payload.recv().await { + match bytes { + Ok(bytes) => { + file.write_all(bytes.as_ref()) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))?; + } + Err(e) => { + return Err(UpdateError::Error(e)); + } + } + } + + file.flush() + .await + .map_err(|e| UpdateError::Error(Box::new(e)))?; + + file.seek(SeekFrom::Start(0)) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))?; + + let mut file = file.into_std().await; + + tokio::task::spawn_blocking(move || { + use std::io::{copy, sink, BufReader, Seek}; + + // If the payload is empty, ignore the check. + if file + .metadata() + .map_err(|e| UpdateError::Error(Box::new(e)))? + .len() + > 0 + { + // Check that the json payload is valid: + let reader = BufReader::new(&mut file); + let mut checker = JsonChecker::new(reader); + + if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() { + // The json file is invalid, we use Serde to get a nice error message: + file.seek(SeekFrom::Start(0)) + .map_err(|e| UpdateError::Error(Box::new(e)))?; + let _: serde_json::Value = serde_json::from_reader(file) + .map_err(|e| UpdateError::Error(Box::new(e)))?; + } + } + + // The payload is valid, we can register it to the update store. + update_store + .register_update(meta, path, uuid) + .map(UpdateStatus::Pending) + .map_err(|e| UpdateError::Error(Box::new(e))) + }) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))? + } + + async fn handle_list_updates(&self, uuid: Uuid) -> Result> { + let update_store = self.store.get(uuid).await?; + tokio::task::spawn_blocking(move || { + let result = update_store + .ok_or(UpdateError::UnexistingIndex(uuid))? + .list() + .map_err(|e| UpdateError::Error(e.into()))?; + Ok(result) + }) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))? + } + + async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { + let store = self + .store + .get(uuid) + .await? + .ok_or(UpdateError::UnexistingIndex(uuid))?; + let result = store + .meta(id) + .map_err(|e| UpdateError::Error(Box::new(e)))? + .ok_or(UpdateError::UnexistingUpdate(id))?; + Ok(result) + } + + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + let store = self.store.delete(uuid).await?; + + if let Some(store) = store { + tokio::task::spawn(async move { + let store = get_arc_ownership_blocking(store).await; + tokio::task::spawn_blocking(move || { + store.prepare_for_closing().wait(); + info!("Update store {} was closed.", uuid); + }); + }); + } + + Ok(()) + } + + async fn handle_create(&self, uuid: Uuid) -> Result<()> { + let _ = self.store.get_or_create(uuid).await?; + Ok(()) + } + + async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let index_handle = self.index_handle.clone(); + if let Some(update_store) = self.store.get(uuid).await? { + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + // acquire write lock to prevent further writes during snapshot + // the update lock must be acquired BEFORE the write lock to prevent dead lock + let _lock = update_store.update_lock.lock(); + let mut txn = update_store.env.write_txn()?; + + // create db snapshot + update_store.snapshot(&mut txn, &path, uuid)?; + + futures::executor::block_on( + async move { index_handle.snapshot(uuid, path).await }, + )?; + Ok(()) + }) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?; + } + + Ok(()) + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs new file mode 100644 index 000000000..43b2ff8c2 --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -0,0 +1,96 @@ +use std::path::{Path, PathBuf}; + +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; + +use super::{ + MapUpdateStoreStore, PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, + UpdateMsg, UpdateStatus, +}; +use crate::index_controller::IndexActorHandle; + +#[derive(Clone)] +pub struct UpdateActorHandleImpl { + sender: mpsc::Sender>, +} + +impl UpdateActorHandleImpl +where + D: AsRef<[u8]> + Sized + 'static + Sync + Send, +{ + pub fn new( + index_handle: I, + path: impl AsRef, + update_store_size: usize, + ) -> anyhow::Result + where + I: IndexActorHandle + 'static, + { + let path = path.as_ref().to_owned().join("updates"); + let (sender, receiver) = mpsc::channel(100); + let store = MapUpdateStoreStore::new(index_handle.clone(), &path, update_store_size); + let actor = UpdateActor::new(store, receiver, path, index_handle)?; + + tokio::task::spawn(actor.run()); + + Ok(Self { sender }) + } +} +#[async_trait::async_trait] +impl UpdateActorHandle for UpdateActorHandleImpl +where + D: AsRef<[u8]> + Sized + 'static + Sync + Send, +{ + type Data = D; + + async fn update( + &self, + meta: UpdateMeta, + data: mpsc::Receiver>, + uuid: Uuid, + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Update { + uuid, + data, + meta, + ret, + }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::ListUpdates { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + + async fn update_status(&self, uuid: Uuid, id: u64) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::GetUpdate { uuid, id, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + + async fn delete(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Delete { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + + async fn create(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Create { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Snapshot { uuid, path, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs new file mode 100644 index 000000000..628a1ebcb --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -0,0 +1,37 @@ +use std::path::PathBuf; + +use uuid::Uuid; +use tokio::sync::{oneshot, mpsc}; + +use super::{Result, PayloadData, UpdateStatus, UpdateMeta}; + +pub enum UpdateMsg { + Update { + uuid: Uuid, + meta: UpdateMeta, + data: mpsc::Receiver>, + ret: oneshot::Sender>, + }, + ListUpdates { + uuid: Uuid, + ret: oneshot::Sender>>, + }, + GetUpdate { + uuid: Uuid, + ret: oneshot::Sender>, + id: u64, + }, + Delete { + uuid: Uuid, + ret: oneshot::Sender>, + }, + Create { + uuid: Uuid, + ret: oneshot::Sender>, + }, + Snapshot { + uuid: Uuid, + path: PathBuf, + ret: oneshot::Sender>, + }, +} diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs new file mode 100644 index 000000000..740323cdc --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -0,0 +1,51 @@ +mod actor; +mod store; +mod message; +mod handle_impl; +mod update_store; + +use std::path::PathBuf; + +use thiserror::Error; +use tokio::sync::mpsc; +use uuid::Uuid; + +use crate::index::UpdateResult; +use crate::index_controller::{UpdateMeta, UpdateStatus}; + +use actor::UpdateActor; +use message::UpdateMsg; +use store::{UpdateStoreStore, MapUpdateStoreStore}; + +pub use handle_impl::UpdateActorHandleImpl; + +pub type Result = std::result::Result; +type UpdateStore = update_store::UpdateStore; +type PayloadData = std::result::Result>; + +#[derive(Debug, Error)] +pub enum UpdateError { + #[error("error with update: {0}")] + Error(Box), + #[error("Index {0} doesn't exist.")] + UnexistingIndex(Uuid), + #[error("Update {0} doesn't exist.")] + UnexistingUpdate(u64), +} + +#[async_trait::async_trait] +pub trait UpdateActorHandle { + type Data: AsRef<[u8]> + Sized + 'static + Sync + Send; + + async fn get_all_updates_status(&self, uuid: Uuid) -> Result>; + async fn update_status(&self, uuid: Uuid, id: u64) -> Result; + async fn delete(&self, uuid: Uuid) -> Result<()>; + async fn create(&self, uuid: Uuid) -> Result<()>; + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn update( + &self, + meta: UpdateMeta, + data: mpsc::Receiver>, + uuid: Uuid, + ) -> Result ; +} diff --git a/meilisearch-http/src/index_controller/update_actor/store.rs b/meilisearch-http/src/index_controller/update_actor/store.rs new file mode 100644 index 000000000..9320c488f --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/store.rs @@ -0,0 +1,112 @@ +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use uuid::Uuid; +use tokio::sync::RwLock; +use tokio::fs; + +use crate::index_controller::IndexActorHandle; +use super::{UpdateStore, UpdateError, Result}; + +#[async_trait::async_trait] +pub trait UpdateStoreStore { + async fn get_or_create(&self, uuid: Uuid) -> Result>; + async fn delete(&self, uuid: Uuid) -> Result>>; + async fn get(&self, uuid: Uuid) -> Result>>; +} + +pub struct MapUpdateStoreStore { + db: Arc>>>, + index_handle: I, + path: PathBuf, + update_store_size: usize, +} + +impl MapUpdateStoreStore { + pub fn new( + index_handle: I, + path: impl AsRef, + update_store_size: usize, + ) -> Self { + let db = Arc::new(RwLock::new(HashMap::new())); + let path = path.as_ref().to_owned(); + Self { + db, + index_handle, + path, + update_store_size, + } + } +} + +#[async_trait::async_trait] +impl UpdateStoreStore for MapUpdateStoreStore { + async fn get_or_create(&self, uuid: Uuid) -> Result> { + match self.db.write().await.entry(uuid) { + Entry::Vacant(e) => { + let mut options = heed::EnvOpenOptions::new(); + let update_store_size = self.update_store_size; + options.map_size(update_store_size); + let path = self.path.clone().join(format!("updates-{}", e.key())); + fs::create_dir_all(&path).await.unwrap(); + let index_handle = self.index_handle.clone(); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle.update(meta, file)) + }) + .map_err(|e| UpdateError::Error(e.into()))?; + let store = e.insert(store); + Ok(store.clone()) + } + Entry::Occupied(e) => Ok(e.get().clone()), + } + } + + async fn get(&self, uuid: Uuid) -> Result>> { + let guard = self.db.read().await; + match guard.get(&uuid) { + Some(uuid) => Ok(Some(uuid.clone())), + None => { + // The index is not found in the found in the loaded indexes, so we attempt to load + // it from disk. We need to acquire a write lock **before** attempting to open the + // index, because someone could be trying to open it at the same time as us. + drop(guard); + let path = self.path.clone().join(format!("updates-{}", uuid)); + if path.exists() { + let mut guard = self.db.write().await; + match guard.entry(uuid) { + Entry::Vacant(entry) => { + // We can safely load the index + let index_handle = self.index_handle.clone(); + let mut options = heed::EnvOpenOptions::new(); + let update_store_size = self.update_store_size; + options.map_size(update_store_size); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle.update(meta, file)) + }) + .map_err(|e| UpdateError::Error(e.into()))?; + let store = entry.insert(store); + Ok(Some(store.clone())) + } + Entry::Occupied(entry) => { + // The index was loaded while we attempted to to iter + Ok(Some(entry.get().clone())) + } + } + } else { + Ok(None) + } + } + } + } + + async fn delete(&self, uuid: Uuid) -> Result>> { + let store = self.db.write().await.remove(&uuid); + let path = self.path.clone().join(format!("updates-{}", uuid)); + if store.is_some() || path.exists() { + fs::remove_dir_all(path).await.unwrap(); + } + Ok(store) + } +} diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs similarity index 100% rename from meilisearch-http/src/index_controller/update_store.rs rename to meilisearch-http/src/index_controller/update_actor/update_store.rs diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs deleted file mode 100644 index 88b62fe1f..000000000 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ /dev/null @@ -1,343 +0,0 @@ -use std::fs::create_dir_all; -use std::path::{Path, PathBuf}; - -use heed::{ - types::{ByteSlice, Str}, - Database, Env, EnvOpenOptions,CompactionOption -}; -use log::{info, warn}; -use thiserror::Error; -use tokio::sync::{mpsc, oneshot}; -use uuid::Uuid; - -const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB - -pub type Result = std::result::Result; - -#[derive(Debug)] -enum UuidResolveMsg { - Resolve { - uid: String, - ret: oneshot::Sender>, - }, - GetOrCreate { - uid: String, - ret: oneshot::Sender>, - }, - Create { - uid: String, - ret: oneshot::Sender>, - }, - Delete { - uid: String, - ret: oneshot::Sender>, - }, - List { - ret: oneshot::Sender>>, - }, - SnapshotRequest { - path: PathBuf, - ret: oneshot::Sender>>, - }, -} - -struct UuidResolverActor { - inbox: mpsc::Receiver, - store: S, -} - -impl UuidResolverActor { - fn new(inbox: mpsc::Receiver, store: S) -> Self { - Self { inbox, store } - } - - async fn run(mut self) { - use UuidResolveMsg::*; - - info!("uuid resolver started"); - - loop { - match self.inbox.recv().await { - Some(Create { uid: name, ret }) => { - let _ = ret.send(self.handle_create(name).await); - } - Some(GetOrCreate { uid: name, ret }) => { - let _ = ret.send(self.handle_get_or_create(name).await); - } - Some(Resolve { uid: name, ret }) => { - let _ = ret.send(self.handle_resolve(name).await); - } - Some(Delete { uid: name, ret }) => { - let _ = ret.send(self.handle_delete(name).await); - } - Some(List { ret }) => { - let _ = ret.send(self.handle_list().await); - } - Some(SnapshotRequest { path, ret }) => { - let _ = ret.send(self.handle_snapshot(path).await); - } - // all senders have been dropped, need to quit. - None => break, - } - } - - warn!("exiting uuid resolver loop"); - } - - async fn handle_create(&self, uid: String) -> Result { - if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)); - } - self.store.create_uuid(uid, true).await - } - - async fn handle_get_or_create(&self, uid: String) -> Result { - if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)); - } - self.store.create_uuid(uid, false).await - } - - async fn handle_resolve(&self, uid: String) -> Result { - self.store - .get_uuid(uid.clone()) - .await? - .ok_or(UuidError::UnexistingIndex(uid)) - } - - async fn handle_delete(&self, uid: String) -> Result { - self.store - .delete(uid.clone()) - .await? - .ok_or(UuidError::UnexistingIndex(uid)) - } - - async fn handle_list(&self) -> Result> { - let result = self.store.list().await?; - Ok(result) - } - - async fn handle_snapshot(&self, path: PathBuf) -> Result> { - self.store.snapshot(path).await - } -} - -fn is_index_uid_valid(uid: &str) -> bool { - uid.chars() - .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') -} - -#[derive(Clone)] -pub struct UuidResolverHandle { - sender: mpsc::Sender, -} - -impl UuidResolverHandle { - pub fn new(path: impl AsRef) -> anyhow::Result { - let (sender, reveiver) = mpsc::channel(100); - let store = HeedUuidStore::new(path)?; - let actor = UuidResolverActor::new(reveiver, store); - tokio::spawn(actor.run()); - Ok(Self { sender }) - } - - pub async fn resolve(&self, name: String) -> anyhow::Result { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Resolve { uid: name, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - - pub async fn get_or_create(&self, name: String) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::GetOrCreate { uid: name, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - - pub async fn create(&self, name: String) -> anyhow::Result { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Create { uid: name, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - - pub async fn delete(&self, name: String) -> anyhow::Result { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Delete { uid: name, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - - pub async fn list(&self) -> anyhow::Result> { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::List { ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - - pub async fn snapshot(&self, path: PathBuf) -> Result> { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::SnapshotRequest { path, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } -} - -#[derive(Debug, Error)] -pub enum UuidError { - #[error("Name already exist.")] - NameAlreadyExist, - #[error("Index \"{0}\" doesn't exist.")] - UnexistingIndex(String), - #[error("Error performing task: {0}")] - TokioTask(#[from] tokio::task::JoinError), - #[error("Database error: {0}")] - Heed(#[from] heed::Error), - #[error("Uuid error: {0}")] - Uuid(#[from] uuid::Error), - #[error("Badly formatted index uid: {0}")] - BadlyFormatted(String), -} - -#[async_trait::async_trait] -trait UuidStore { - // Create a new entry for `name`. Return an error if `err` and the entry already exists, return - // the uuid otherwise. - async fn create_uuid(&self, uid: String, err: bool) -> Result; - async fn get_uuid(&self, uid: String) -> Result>; - async fn delete(&self, uid: String) -> Result>; - async fn list(&self) -> Result>; - async fn snapshot(&self, path: PathBuf) -> Result>; -} - -struct HeedUuidStore { - env: Env, - db: Database, -} - -impl HeedUuidStore { - fn new(path: impl AsRef) -> anyhow::Result { - let path = path.as_ref().join("index_uuids"); - create_dir_all(&path)?; - let mut options = EnvOpenOptions::new(); - options.map_size(UUID_STORE_SIZE); // 1GB - let env = options.open(path)?; - let db = env.create_database(None)?; - Ok(Self { env, db }) - } -} - -#[async_trait::async_trait] -impl UuidStore for HeedUuidStore { - async fn create_uuid(&self, name: String, err: bool) -> Result { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let mut txn = env.write_txn()?; - match db.get(&txn, &name)? { - Some(uuid) => { - if err { - Err(UuidError::NameAlreadyExist) - } else { - let uuid = Uuid::from_slice(uuid)?; - Ok(uuid) - } - } - None => { - let uuid = Uuid::new_v4(); - db.put(&mut txn, &name, uuid.as_bytes())?; - txn.commit()?; - Ok(uuid) - } - } - }) - .await? - } - async fn get_uuid(&self, name: String) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let txn = env.read_txn()?; - match db.get(&txn, &name)? { - Some(uuid) => { - let uuid = Uuid::from_slice(uuid)?; - Ok(Some(uuid)) - } - None => Ok(None), - } - }) - .await? - } - - async fn delete(&self, uid: String) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let mut txn = env.write_txn()?; - match db.get(&txn, &uid)? { - Some(uuid) => { - let uuid = Uuid::from_slice(uuid)?; - db.delete(&mut txn, &uid)?; - txn.commit()?; - Ok(Some(uuid)) - } - None => Ok(None), - } - }) - .await? - } - - async fn list(&self) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let txn = env.read_txn()?; - let mut entries = Vec::new(); - for entry in db.iter(&txn)? { - let (name, uuid) = entry?; - let uuid = Uuid::from_slice(uuid)?; - entries.push((name.to_owned(), uuid)) - } - Ok(entries) - }) - .await? - } - - async fn snapshot(&self, mut path: PathBuf) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - // Write transaction to acquire a lock on the database. - let txn = env.write_txn()?; - let mut entries = Vec::new(); - for entry in db.iter(&txn)? { - let (_, uuid) = entry?; - let uuid = Uuid::from_slice(uuid)?; - entries.push(uuid) - } - - // only perform snapshot if there are indexes - if !entries.is_empty() { - path.push("index_uuids"); - create_dir_all(&path).unwrap(); - path.push("data.mdb"); - env.copy_to_path(path, CompactionOption::Enabled)?; - } - Ok(entries) - }) - .await? - } -} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs new file mode 100644 index 000000000..4c0a13ad7 --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -0,0 +1,94 @@ +use std::path::PathBuf; + +use log::{info, warn}; +use tokio::sync::mpsc; +use uuid::Uuid; + +use super::{UuidResolveMsg, UuidStore, Result, UuidError}; + +pub struct UuidResolverActor { + inbox: mpsc::Receiver, + store: S, +} + +impl UuidResolverActor { + pub fn new(inbox: mpsc::Receiver, store: S) -> Self { + Self { inbox, store } + } + + pub async fn run(mut self) { + use UuidResolveMsg::*; + + info!("uuid resolver started"); + + loop { + match self.inbox.recv().await { + Some(Create { uid: name, ret }) => { + let _ = ret.send(self.handle_create(name).await); + } + Some(GetOrCreate { uid: name, ret }) => { + let _ = ret.send(self.handle_get_or_create(name).await); + } + Some(Resolve { uid: name, ret }) => { + let _ = ret.send(self.handle_resolve(name).await); + } + Some(Delete { uid: name, ret }) => { + let _ = ret.send(self.handle_delete(name).await); + } + Some(List { ret }) => { + let _ = ret.send(self.handle_list().await); + } + Some(SnapshotRequest { path, ret }) => { + let _ = ret.send(self.handle_snapshot(path).await); + } + // all senders have been dropped, need to quit. + None => break, + } + } + + warn!("exiting uuid resolver loop"); + } + + async fn handle_create(&self, uid: String) -> Result { + if !is_index_uid_valid(&uid) { + return Err(UuidError::BadlyFormatted(uid)); + } + self.store.create_uuid(uid, true).await + } + + async fn handle_get_or_create(&self, uid: String) -> Result { + if !is_index_uid_valid(&uid) { + return Err(UuidError::BadlyFormatted(uid)); + } + self.store.create_uuid(uid, false).await + } + + async fn handle_resolve(&self, uid: String) -> Result { + self.store + .get_uuid(uid.clone()) + .await? + .ok_or(UuidError::UnexistingIndex(uid)) + } + + async fn handle_delete(&self, uid: String) -> Result { + self.store + .delete(uid.clone()) + .await? + .ok_or(UuidError::UnexistingIndex(uid)) + } + + async fn handle_list(&self) -> Result> { + let result = self.store.list().await?; + Ok(result) + } + + async fn handle_snapshot(&self, path: PathBuf) -> Result> { + self.store.snapshot(path).await + } +} + +fn is_index_uid_valid(uid: &str) -> bool { + uid.chars() + .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') +} + diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs new file mode 100644 index 000000000..265ea8422 --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -0,0 +1,78 @@ +use std::path::{Path, PathBuf}; + +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; + +use super::{HeedUuidStore, UuidResolverActor, UuidResolveMsg, UuidResolverHandle, Result}; + +#[derive(Clone)] +pub struct UuidResolverHandleImpl { + sender: mpsc::Sender, +} + +impl UuidResolverHandleImpl { + pub fn new(path: impl AsRef) -> anyhow::Result { + let (sender, reveiver) = mpsc::channel(100); + let store = HeedUuidStore::new(path)?; + let actor = UuidResolverActor::new(reveiver, store); + tokio::spawn(actor.run()); + Ok(Self { sender }) + } +} + +#[async_trait::async_trait] +impl UuidResolverHandle for UuidResolverHandleImpl { + async fn resolve(&self, name: String) -> anyhow::Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Resolve { uid: name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn get_or_create(&self, name: String) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::GetOrCreate { uid: name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn create(&self, name: String) -> anyhow::Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Create { uid: name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn delete(&self, name: String) -> anyhow::Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Delete { uid: name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn list(&self) -> anyhow::Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::List { ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn snapshot(&self, path: PathBuf) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::SnapshotRequest { path, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs new file mode 100644 index 000000000..0e8323a82 --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -0,0 +1,33 @@ +use std::path::PathBuf; + +use tokio::sync::oneshot; +use uuid::Uuid; + +use super::Result; + +#[derive(Debug)] +pub enum UuidResolveMsg { + Resolve { + uid: String, + ret: oneshot::Sender>, + }, + GetOrCreate { + uid: String, + ret: oneshot::Sender>, + }, + Create { + uid: String, + ret: oneshot::Sender>, + }, + Delete { + uid: String, + ret: oneshot::Sender>, + }, + List { + ret: oneshot::Sender>>, + }, + SnapshotRequest { + path: PathBuf, + ret: oneshot::Sender>>, + }, +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs new file mode 100644 index 000000000..08cbe70e0 --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -0,0 +1,45 @@ +mod actor; +mod handle_impl; +mod message; +mod store; + +use std::path::PathBuf; + +use thiserror::Error; +use uuid::Uuid; + +use actor::UuidResolverActor; +use message::UuidResolveMsg; +use store::{HeedUuidStore, UuidStore}; + +pub use handle_impl::UuidResolverHandleImpl; + +const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB + +pub type Result = std::result::Result; + +#[async_trait::async_trait] +pub trait UuidResolverHandle { + async fn resolve(&self, name: String) -> anyhow::Result; + async fn get_or_create(&self, name: String) -> Result; + async fn create(&self, name: String) -> anyhow::Result; + async fn delete(&self, name: String) -> anyhow::Result; + async fn list(&self) -> anyhow::Result>; + async fn snapshot(&self, path: PathBuf) -> Result>; +} + +#[derive(Debug, Error)] +pub enum UuidError { + #[error("Name already exist.")] + NameAlreadyExist, + #[error("Index \"{0}\" doesn't exist.")] + UnexistingIndex(String), + #[error("Error performing task: {0}")] + TokioTask(#[from] tokio::task::JoinError), + #[error("Database error: {0}")] + Heed(#[from] heed::Error), + #[error("Uuid error: {0}")] + Uuid(#[from] uuid::Error), + #[error("Badly formatted index uid: {0}")] + BadlyFormatted(String), +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs new file mode 100644 index 000000000..e821f2b2b --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -0,0 +1,140 @@ +use std::fs::create_dir_all; +use std::path::{Path, PathBuf}; + +use heed::{ + types::{ByteSlice, Str}, + Database, Env, EnvOpenOptions,CompactionOption +}; +use uuid::Uuid; + +use super::{UUID_STORE_SIZE, UuidError, Result}; + +#[async_trait::async_trait] +pub trait UuidStore { + // Create a new entry for `name`. Return an error if `err` and the entry already exists, return + // the uuid otherwise. + async fn create_uuid(&self, uid: String, err: bool) -> Result; + async fn get_uuid(&self, uid: String) -> Result>; + async fn delete(&self, uid: String) -> Result>; + async fn list(&self) -> Result>; + async fn snapshot(&self, path: PathBuf) -> Result>; +} + +pub struct HeedUuidStore { + env: Env, + db: Database, +} + +impl HeedUuidStore { + pub fn new(path: impl AsRef) -> anyhow::Result { + let path = path.as_ref().join("index_uuids"); + create_dir_all(&path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(UUID_STORE_SIZE); // 1GB + let env = options.open(path)?; + let db = env.create_database(None)?; + Ok(Self { env, db }) + } +} + +#[async_trait::async_trait] +impl UuidStore for HeedUuidStore { + async fn create_uuid(&self, name: String, err: bool) -> Result { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + let mut txn = env.write_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + if err { + Err(UuidError::NameAlreadyExist) + } else { + let uuid = Uuid::from_slice(uuid)?; + Ok(uuid) + } + } + None => { + let uuid = Uuid::new_v4(); + db.put(&mut txn, &name, uuid.as_bytes())?; + txn.commit()?; + Ok(uuid) + } + } + }) + .await? + } + async fn get_uuid(&self, name: String) -> Result> { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + let txn = env.read_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + Ok(Some(uuid)) + } + None => Ok(None), + } + }) + .await? + } + + async fn delete(&self, uid: String) -> Result> { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + let mut txn = env.write_txn()?; + match db.get(&txn, &uid)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + db.delete(&mut txn, &uid)?; + txn.commit()?; + Ok(Some(uuid)) + } + None => Ok(None), + } + }) + .await? + } + + async fn list(&self) -> Result> { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + let txn = env.read_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (name, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push((name.to_owned(), uuid)) + } + Ok(entries) + }) + .await? + } + + async fn snapshot(&self, mut path: PathBuf) -> Result> { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + // Write transaction to acquire a lock on the database. + let txn = env.write_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (_, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push(uuid) + } + + // only perform snapshot if there are indexes + if !entries.is_empty() { + path.push("index_uuids"); + create_dir_all(&path).unwrap(); + path.push("data.mdb"); + env.copy_to_path(path, CompactionOption::Enabled)?; + } + Ok(entries) + }) + .await? + } +} diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 43caf1dc6..fb5147811 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -43,7 +43,7 @@ impl Server { ignore_snapshot_if_db_exists: false, snapshot_dir: ".".into(), schedule_snapshot: false, - snapshot_interval_sec: None, + snapshot_interval_sec: 0, import_dump: None, indexer_options: IndexerOpts::default(), #[cfg(all(not(debug_assertions), feature = "sentry"))] From 46293546f3ec473e9d85be4a4e862153637e7083 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Mar 2021 16:19:01 +0100 Subject: [PATCH 219/522] add tests and mocks --- Cargo.lock | 96 +++++++++++ meilisearch-http/Cargo.toml | 5 +- .../src/index_controller/index_actor/mod.rs | 6 +- .../src/index_controller/snapshot.rs | 158 +++++++++++++++++- .../index_controller/update_actor/actor.rs | 2 +- .../update_actor/handle_impl.rs | 2 +- .../src/index_controller/update_actor/mod.rs | 4 + .../index_controller/update_actor/store.rs | 19 +-- .../src/index_controller/uuid_resolver/mod.rs | 4 + 9 files changed, 273 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dfb58ff89..afbb6eb0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -937,6 +937,12 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + [[package]] name = "digest" version = "0.8.1" @@ -961,6 +967,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "downcast" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" + [[package]] name = "either" version = "1.6.1" @@ -1066,6 +1078,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1082,6 +1103,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" + [[package]] name = "fs_extra" version = "1.2.0" @@ -1822,6 +1849,7 @@ dependencies = [ "memmap", "milli", "mime", + "mockall", "once_cell", "oxidized-json-checker", "parking_lot", @@ -2023,6 +2051,33 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "mockall" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d614ad23f9bb59119b8b5670a85c7ba92c5e9adf4385c81ea00c51c8be33d5" +dependencies = [ + "cfg-if 1.0.0", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd4234635bca06fc96c7368d038061e0aae1b00a764dc817e900dc974e3deea" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.64", +] + [[package]] name = "net2" version = "0.2.37" @@ -2046,6 +2101,12 @@ dependencies = [ "libc", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "ntapi" version = "0.3.6" @@ -2329,6 +2390,35 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "predicates" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" + +[[package]] +name = "predicates-tree" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" +dependencies = [ + "predicates-core", + "treeline", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3438,6 +3528,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "treeline" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" + [[package]] name = "trust-dns-proto" version = "0.19.7" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 4b8ecd13d..93776269f 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -80,10 +80,11 @@ version = "0.18.1" [dev-dependencies] +actix-rt = "2.1.0" +assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } +mockall = "0.9.1" serde_url_params = "0.2.0" tempdir = "0.3.7" -assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } -actix-rt = "2.1.0" urlencoding = "1.1.1" [features] diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index dac7ef06c..e3e48add7 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -23,6 +23,9 @@ use actor::IndexActor; pub use handle_impl::IndexActorHandleImpl; +#[cfg(test)] +use mockall::automock; + pub type Result = std::result::Result; type UpdateResult = std::result::Result, Failed>; @@ -68,7 +71,8 @@ pub enum IndexError { #[async_trait::async_trait] -pub trait IndexActorHandle: Sync + Send + Clone { +#[cfg_attr(test, automock)] +pub trait IndexActorHandle { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn update( &self, diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index afdcdaf23..998821d9f 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -7,9 +7,9 @@ use tokio::fs; use tokio::task::spawn_blocking; use tokio::time::sleep; -use crate::helpers::compression; use super::update_actor::UpdateActorHandle; use super::uuid_resolver::UuidResolverHandle; +use crate::helpers::compression; #[allow(dead_code)] pub struct SnapshotService { @@ -22,7 +22,7 @@ pub struct SnapshotService { impl SnapshotService where U: UpdateActorHandle, - R: UuidResolverHandle + R: UuidResolverHandle, { pub fn new( uuid_resolver_handle: R, @@ -39,7 +39,6 @@ where } pub async fn run(self) { - loop { sleep(self.snapshot_period).await; if let Err(e) = self.perform_snapshot().await { @@ -49,7 +48,7 @@ where } async fn perform_snapshot(&self) -> anyhow::Result<()> { - if self.snapshot_path.file_name().is_none() { + if !self.snapshot_path.is_file() { bail!("invalid snapshot file path"); } @@ -58,15 +57,21 @@ where fs::create_dir_all(&temp_snapshot_path).await?; - let uuids = self.uuid_resolver_handle.snapshot(temp_snapshot_path.clone()).await?; + let uuids = self + .uuid_resolver_handle + .snapshot(temp_snapshot_path.clone()) + .await?; if uuids.is_empty() { - return Ok(()) + return Ok(()); } let tasks = uuids .iter() - .map(|&uuid| self.update_handle.snapshot(uuid, temp_snapshot_path.clone())) + .map(|&uuid| { + self.update_handle + .snapshot(uuid, temp_snapshot_path.clone()) + }) .collect::>(); futures::future::try_join_all(tasks).await?; @@ -75,7 +80,10 @@ where let temp_snapshot_file_clone = temp_snapshot_file.clone(); let temp_snapshot_path_clone = temp_snapshot_path.clone(); - spawn_blocking(move || compression::to_tar_gz(temp_snapshot_path_clone, temp_snapshot_file_clone)).await??; + spawn_blocking(move || { + compression::to_tar_gz(temp_snapshot_path_clone, temp_snapshot_file_clone) + }) + .await??; fs::rename(temp_snapshot_file, &self.snapshot_path).await?; @@ -84,3 +92,137 @@ where Ok(()) } } + +#[cfg(test)] +mod test { + use futures::future::{ok, err}; + use rand::Rng; + use tokio::time::timeout; + use uuid::Uuid; + + use super::*; + use crate::index_controller::update_actor::{MockUpdateActorHandle, UpdateError}; + use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidError}; + + #[actix_rt::test] + async fn test_normal() { + let mut rng = rand::thread_rng(); + let uuids_num = rng.gen_range(5, 10); + let uuids = (0..uuids_num).map(|_| Uuid::new_v4()).collect::>(); + + let mut uuid_resolver = MockUuidResolverHandle::new(); + let uuids_clone = uuids.clone(); + uuid_resolver + .expect_snapshot() + .times(1) + .returning(move |_| Box::pin(ok(uuids_clone.clone()))); + + let mut update_handle = MockUpdateActorHandle::new(); + let uuids_clone = uuids.clone(); + update_handle + .expect_snapshot() + .withf(move |uuid, _path| uuids_clone.contains(uuid)) + .times(uuids_num) + .returning(move |_, _| Box::pin(ok(()))); + + let snapshot_path = tempfile::NamedTempFile::new_in(".").unwrap(); + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + snapshot_path.path().to_owned(), + ); + + snapshot_service.perform_snapshot().await.unwrap(); + } + + #[actix_rt::test] + async fn bad_file_name() { + let uuid_resolver = MockUuidResolverHandle::new(); + let update_handle = MockUpdateActorHandle::new(); + + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + "directory/".into(), + ); + + assert!(snapshot_service.perform_snapshot().await.is_err()); + } + + #[actix_rt::test] + async fn error_performing_uuid_snapshot() { + let mut uuid_resolver = MockUuidResolverHandle::new(); + uuid_resolver + .expect_snapshot() + .times(1) + // abitrary error + .returning(|_| Box::pin(err(UuidError::NameAlreadyExist))); + + let update_handle = MockUpdateActorHandle::new(); + + let snapshot_path = tempfile::NamedTempFile::new_in(".").unwrap(); + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + snapshot_path.path().to_owned(), + ); + + assert!(snapshot_service.perform_snapshot().await.is_err()); + // Nothing was written to the file + assert_eq!(snapshot_path.as_file().metadata().unwrap().len(), 0); + } + + #[actix_rt::test] + async fn error_performing_index_snapshot() { + let uuid = Uuid::new_v4(); + let mut uuid_resolver = MockUuidResolverHandle::new(); + uuid_resolver + .expect_snapshot() + .times(1) + .returning(move |_| Box::pin(ok(vec![uuid]))); + + let mut update_handle = MockUpdateActorHandle::new(); + update_handle + .expect_snapshot() + // abitrary error + .returning(|_, _| Box::pin(err(UpdateError::UnexistingUpdate(0)))); + + let snapshot_path = tempfile::NamedTempFile::new_in(".").unwrap(); + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + snapshot_path.path().to_owned(), + ); + + assert!(snapshot_service.perform_snapshot().await.is_err()); + // Nothing was written to the file + assert_eq!(snapshot_path.as_file().metadata().unwrap().len(), 0); + } + + #[actix_rt::test] + async fn test_loop() { + let mut uuid_resolver = MockUuidResolverHandle::new(); + uuid_resolver + .expect_snapshot() + // we expect the funtion to be called between 2 and 3 time in the given interval. + .times(2..4) + // abitrary error, to short-circuit the function + .returning(move |_| Box::pin(err(UuidError::NameAlreadyExist))); + + let update_handle = MockUpdateActorHandle::new(); + + let snapshot_path = tempfile::NamedTempFile::new_in(".").unwrap(); + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + snapshot_path.path().to_owned(), + ); + + let _ = timeout(Duration::from_millis(300), snapshot_service.run()).await; + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 629a87ba4..df949279e 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -23,7 +23,7 @@ impl UpdateActor where D: AsRef<[u8]> + Sized + 'static, S: UpdateStoreStore, - I: IndexActorHandle + 'static, + I: IndexActorHandle + Clone + Send + Sync + 'static, { pub fn new( store: S, diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 43b2ff8c2..59f67fbe0 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -24,7 +24,7 @@ where update_store_size: usize, ) -> anyhow::Result where - I: IndexActorHandle + 'static, + I: IndexActorHandle + Clone + Send + Sync + 'static, { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 740323cdc..1fb53a2b3 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -23,6 +23,9 @@ pub type Result = std::result::Result; type UpdateStore = update_store::UpdateStore; type PayloadData = std::result::Result>; +#[cfg(test)] +use mockall::automock; + #[derive(Debug, Error)] pub enum UpdateError { #[error("error with update: {0}")] @@ -34,6 +37,7 @@ pub enum UpdateError { } #[async_trait::async_trait] +#[cfg_attr(test, automock(type Data=Vec;))] pub trait UpdateActorHandle { type Data: AsRef<[u8]> + Sized + 'static + Sync + Send; diff --git a/meilisearch-http/src/index_controller/update_actor/store.rs b/meilisearch-http/src/index_controller/update_actor/store.rs index 9320c488f..676182a62 100644 --- a/meilisearch-http/src/index_controller/update_actor/store.rs +++ b/meilisearch-http/src/index_controller/update_actor/store.rs @@ -1,14 +1,14 @@ -use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; -use uuid::Uuid; -use tokio::sync::RwLock; use tokio::fs; +use tokio::sync::RwLock; +use uuid::Uuid; +use super::{Result, UpdateError, UpdateStore}; use crate::index_controller::IndexActorHandle; -use super::{UpdateStore, UpdateError, Result}; #[async_trait::async_trait] pub trait UpdateStoreStore { @@ -25,11 +25,7 @@ pub struct MapUpdateStoreStore { } impl MapUpdateStoreStore { - pub fn new( - index_handle: I, - path: impl AsRef, - update_store_size: usize, - ) -> Self { + pub fn new(index_handle: I, path: impl AsRef, update_store_size: usize) -> Self { let db = Arc::new(RwLock::new(HashMap::new())); let path = path.as_ref().to_owned(); Self { @@ -42,7 +38,10 @@ impl MapUpdateStoreStore { } #[async_trait::async_trait] -impl UpdateStoreStore for MapUpdateStoreStore { +impl UpdateStoreStore for MapUpdateStoreStore +where + I: IndexActorHandle + Clone + Send + Sync + 'static, +{ async fn get_or_create(&self, uuid: Uuid) -> Result> { match self.db.write().await.entry(uuid) { Entry::Vacant(e) => { diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 08cbe70e0..2361fc08b 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -12,6 +12,9 @@ use actor::UuidResolverActor; use message::UuidResolveMsg; use store::{HeedUuidStore, UuidStore}; +#[cfg(test)] +use mockall::automock; + pub use handle_impl::UuidResolverHandleImpl; const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB @@ -19,6 +22,7 @@ const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB pub type Result = std::result::Result; #[async_trait::async_trait] +#[cfg_attr(test, automock)] pub trait UuidResolverHandle { async fn resolve(&self, name: String) -> anyhow::Result; async fn get_or_create(&self, name: String) -> Result; From eb53ed4cc14312e65e389c8a0183547b1932e15c Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Mar 2021 16:37:46 +0100 Subject: [PATCH 220/522] load snapshot --- meilisearch-http/src/helpers/compression.rs | 6 ++-- meilisearch-http/src/index_controller/mod.rs | 10 ++++-- .../src/index_controller/snapshot.rs | 33 +++++++++++++++++-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index 74578ed0c..201f35149 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -5,9 +5,7 @@ use std::path::Path; use flate2::{Compression, write::GzEncoder, read::GzDecoder}; use tar::{Archive, Builder}; -use crate::error::Error; - -pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Error> { +pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { let mut f = File::create(dest)?; let gz_encoder = GzEncoder::new(&mut f, Compression::default()); let mut tar_encoder = Builder::new(gz_encoder); @@ -18,7 +16,7 @@ pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Er Ok(()) } -pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Error> { +pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { let f = File::open(&src)?; let gz = GzDecoder::new(f); let mut ar = Archive::new(gz); diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index bcd0cd8fe..73d071daa 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -20,10 +20,11 @@ use tokio::time::sleep; use crate::index::{Document, SearchQuery, SearchResult}; use crate::index::{Facets, Settings, UpdateResult}; use crate::option::Opt; -use crate::helpers::compression; + use index_actor::IndexActorHandle; use update_actor::UpdateActorHandle; use uuid_resolver::UuidResolverHandle; +use snapshot::load_snapshot; use snapshot::SnapshotService; pub use updates::{Failed, Processed, Processing}; @@ -71,7 +72,12 @@ impl IndexController { let update_store_size = options.max_udb_size.get_bytes() as usize; if let Some(ref path) = options.import_snapshot { - compression::from_tar_gz(path, &options.db_path)?; + load_snapshot( + &options.db_path, + path, + options.ignore_snapshot_if_db_exists, + options.ignore_missing_snapshot, + )?; } let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&path)?; diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 998821d9f..586d096ba 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::bail; @@ -93,9 +93,38 @@ where } } +pub fn load_snapshot( + db_path: impl AsRef, + snapshot_path: impl AsRef, + ignore_snapshot_if_db_exists: bool, + ignore_missing_snapshot: bool, +) -> anyhow::Result<()> { + if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() { + compression::from_tar_gz(snapshot_path, db_path) + } else if db_path.as_ref().exists() && !ignore_snapshot_if_db_exists { + bail!( + "database already exists at {:?}, try to delete it or rename it", + db_path + .as_ref() + .canonicalize() + .unwrap_or(db_path.as_ref().to_owned()) + ) + } else if !snapshot_path.as_ref().exists() && !ignore_missing_snapshot { + bail!( + "snapshot doesn't exist at {:?}", + snapshot_path + .as_ref() + .canonicalize() + .unwrap_or(snapshot_path.as_ref().to_owned()) + ) + } else { + Ok(()) + } +} + #[cfg(test)] mod test { - use futures::future::{ok, err}; + use futures::future::{err, ok}; use rand::Rng; use tokio::time::timeout; use uuid::Uuid; From 48d5f88c1a1869f39f3e1b394c24e6f12fea7829 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Mar 2021 17:23:57 +0100 Subject: [PATCH 221/522] fix snapshot dir already exists --- meilisearch-http/src/data/mod.rs | 2 -- meilisearch-http/src/index_controller/mod.rs | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 19004da70..717d728fc 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -1,7 +1,6 @@ pub mod search; mod updates; -use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; @@ -59,7 +58,6 @@ impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - create_dir_all(&path)?; let index_controller = IndexController::new(&path, &options)?; let mut api_keys = ApiKeys { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 73d071daa..77ad957f8 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -80,6 +80,8 @@ impl IndexController { )?; } + std::fs::create_dir_all(&path)?; + let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&path)?; let index_handle = index_actor::IndexActorHandleImpl::new(&path, index_size)?; let update_handle = update_actor::UpdateActorHandleImpl::new( From 06f9dae0f37634d9c4b1eb7455649f4064edd364 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Mar 2021 17:26:18 +0100 Subject: [PATCH 222/522] remove prints --- .../src/index_controller/snapshot.rs | 2 +- .../update_actor/update_store.rs | 92 ------------------- 2 files changed, 1 insertion(+), 93 deletions(-) diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 586d096ba..e43f521a0 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -49,7 +49,7 @@ where async fn perform_snapshot(&self) -> anyhow::Result<()> { if !self.snapshot_path.is_file() { - bail!("invalid snapshot file path"); + bail!("Invalid snapshot file path."); } let temp_snapshot_dir = spawn_blocking(move || tempfile::tempdir_in(".")).await??; diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index e17c56e10..63806260b 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -401,98 +401,6 @@ where copy(path, to)?; } - println!("done"); - Ok(()) } } - -//#[cfg(test)] -//mod tests { -//use super::*; -//use std::thread; -//use std::time::{Duration, Instant}; - -//#[test] -//fn simple() { -//let dir = tempfile::tempdir().unwrap(); -//let mut options = EnvOpenOptions::new(); -//options.map_size(4096 * 100); -//let update_store = UpdateStore::open( -//options, -//dir, -//|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { -//let new_meta = meta.meta().to_string() + " processed"; -//let processed = meta.process(new_meta); -//Ok(processed) -//}, -//) -//.unwrap(); - -//let meta = String::from("kiki"); -//let update = update_store.register_update(meta, &[]).unwrap(); -//thread::sleep(Duration::from_millis(100)); -//let meta = update_store.meta(update.id()).unwrap().unwrap(); -//if let UpdateStatus::Processed(Processed { success, .. }) = meta { -//assert_eq!(success, "kiki processed"); -//} else { -//panic!() -//} -//} - -//#[test] -//#[ignore] -//fn long_running_update() { -//let dir = tempfile::tempdir().unwrap(); -//let mut options = EnvOpenOptions::new(); -//options.map_size(4096 * 100); -//let update_store = UpdateStore::open( -//options, -//dir, -//|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { -//thread::sleep(Duration::from_millis(400)); -//let new_meta = meta.meta().to_string() + "processed"; -//let processed = meta.process(new_meta); -//Ok(processed) -//}, -//) -//.unwrap(); - -//let before_register = Instant::now(); - -//let meta = String::from("kiki"); -//let update_kiki = update_store.register_update(meta, &[]).unwrap(); -//assert!(before_register.elapsed() < Duration::from_millis(200)); - -//let meta = String::from("coco"); -//let update_coco = update_store.register_update(meta, &[]).unwrap(); -//assert!(before_register.elapsed() < Duration::from_millis(200)); - -//let meta = String::from("cucu"); -//let update_cucu = update_store.register_update(meta, &[]).unwrap(); -//assert!(before_register.elapsed() < Duration::from_millis(200)); - -//thread::sleep(Duration::from_millis(400 * 3 + 100)); - -//let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); -//if let UpdateStatus::Processed(Processed { success, .. }) = meta { -//assert_eq!(success, "kiki processed"); -//} else { -//panic!() -//} - -//let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); -//if let UpdateStatus::Processed(Processed { success, .. }) = meta { -//assert_eq!(success, "coco processed"); -//} else { -//panic!() -//} - -//let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); -//if let UpdateStatus::Processed(Processed { success, .. }) = meta { -//assert_eq!(success, "cucu processed"); -//} else { -//panic!() -//} -//} -//} From 1f16c8d2240d719fd783c54869677e686b2f567f Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Mar 2021 11:03:01 +0100 Subject: [PATCH 223/522] integration test snapshot --- .../src/index_controller/snapshot.rs | 3 + meilisearch-http/src/snapshot_old.rs | 96 ------------------- meilisearch-http/tests/common/mod.rs | 6 +- meilisearch-http/tests/common/server.rs | 81 +++++++++------- meilisearch-http/tests/integration.rs | 3 +- 5 files changed, 55 insertions(+), 134 deletions(-) delete mode 100644 meilisearch-http/src/snapshot_old.rs diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index e43f521a0..98fabcca7 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -39,6 +39,7 @@ where } pub async fn run(self) { + info!("Snashot scheduled every {}s.", self.snapshot_period.as_secs()); loop { sleep(self.snapshot_period).await; if let Err(e) = self.perform_snapshot().await { @@ -52,6 +53,8 @@ where bail!("Invalid snapshot file path."); } + info!("Performing snapshot."); + let temp_snapshot_dir = spawn_blocking(move || tempfile::tempdir_in(".")).await??; let temp_snapshot_path = temp_snapshot_dir.path().to_owned(); diff --git a/meilisearch-http/src/snapshot_old.rs b/meilisearch-http/src/snapshot_old.rs deleted file mode 100644 index ed5cd9a81..000000000 --- a/meilisearch-http/src/snapshot_old.rs +++ /dev/null @@ -1,96 +0,0 @@ -use crate::Data; -use crate::error::Error; -use crate::helpers::compression; - -use log::error; -use std::fs::create_dir_all; -use std::path::Path; -use std::thread; -use std::time::{Duration}; -use tempfile::TempDir; - -pub fn load_snapshot( - db_path: &str, - snapshot_path: &Path, - ignore_snapshot_if_db_exists: bool, - ignore_missing_snapshot: bool -) -> Result<(), Error> { - let db_path = Path::new(db_path); - - if !db_path.exists() && snapshot_path.exists() { - compression::from_tar_gz(snapshot_path, db_path) - } else if db_path.exists() && !ignore_snapshot_if_db_exists { - Err(Error::Internal(format!("database already exists at {:?}, try to delete it or rename it", db_path.canonicalize().unwrap_or(db_path.into())))) - } else if !snapshot_path.exists() && !ignore_missing_snapshot { - Err(Error::Internal(format!("snapshot doesn't exist at {:?}", snapshot_path.canonicalize().unwrap_or(snapshot_path.into())))) - } else { - Ok(()) - } -} - -pub fn create_snapshot(data: &Data, snapshot_path: &Path) -> Result<(), Error> { - let tmp_dir = TempDir::new()?; - - data.db.copy_and_compact_to_path(tmp_dir.path())?; - - compression::to_tar_gz(tmp_dir.path(), snapshot_path).map_err(|e| Error::Internal(format!("something went wrong during snapshot compression: {}", e))) -} - -pub fn schedule_snapshot(data: Data, snapshot_dir: &Path, time_gap_s: u64) -> Result<(), Error> { - if snapshot_dir.file_name().is_none() { - return Err(Error::Internal("invalid snapshot file path".to_string())); - } - let db_name = Path::new(&data.db_path).file_name().ok_or_else(|| Error::Internal("invalid database name".to_string()))?; - create_dir_all(snapshot_dir)?; - let snapshot_path = snapshot_dir.join(format!("{}.snapshot", db_name.to_str().unwrap_or("data.ms"))); - - thread::spawn(move || loop { - if let Err(e) = create_snapshot(&data, &snapshot_path) { - error!("Unsuccessful snapshot creation: {}", e); - } - thread::sleep(Duration::from_secs(time_gap_s)); - }); - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::prelude::*; - use std::fs; - - #[test] - fn test_pack_unpack() { - let tempdir = TempDir::new().unwrap(); - - let test_dir = tempdir.path(); - let src_dir = test_dir.join("src"); - let dest_dir = test_dir.join("complex/destination/path/"); - let archive_path = test_dir.join("archive.snapshot"); - - let file_1_relative = Path::new("file1.txt"); - let subdir_relative = Path::new("subdir/"); - let file_2_relative = Path::new("subdir/file2.txt"); - - create_dir_all(src_dir.join(subdir_relative)).unwrap(); - fs::File::create(src_dir.join(file_1_relative)).unwrap().write_all(b"Hello_file_1").unwrap(); - fs::File::create(src_dir.join(file_2_relative)).unwrap().write_all(b"Hello_file_2").unwrap(); - - - assert!(compression::to_tar_gz(&src_dir, &archive_path).is_ok()); - assert!(archive_path.exists()); - assert!(load_snapshot(&dest_dir.to_str().unwrap(), &archive_path, false, false).is_ok()); - - assert!(dest_dir.exists()); - assert!(dest_dir.join(file_1_relative).exists()); - assert!(dest_dir.join(subdir_relative).exists()); - assert!(dest_dir.join(file_2_relative).exists()); - - let contents = fs::read_to_string(dest_dir.join(file_1_relative)).unwrap(); - assert_eq!(contents, "Hello_file_1"); - - let contents = fs::read_to_string(dest_dir.join(file_2_relative)).unwrap(); - assert_eq!(contents, "Hello_file_2"); - } -} diff --git a/meilisearch-http/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs index d1874ae84..e734b3621 100644 --- a/meilisearch-http/tests/common/mod.rs +++ b/meilisearch-http/tests/common/mod.rs @@ -1,6 +1,6 @@ -mod index; -mod server; -mod service; +pub mod index; +pub mod server; +pub mod service; pub use index::{GetAllDocumentsOptions, GetDocumentOptions}; pub use server::Server; diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index fb5147811..da490250b 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use actix_web::http::StatusCode; use byte_unit::{Byte, ByteUnit}; use serde_json::Value; @@ -12,50 +14,27 @@ use super::service::Service; pub struct Server { pub service: Service, - // hod ownership to the tempdir while we use the server instance. - _dir: tempdir::TempDir, + // hold ownership to the tempdir while we use the server instance. + _dir: Option, } impl Server { pub async fn new() -> Self { let dir = TempDir::new("meilisearch").unwrap(); - let opt = Opt { - db_path: dir.path().join("db"), - dumps_dir: dir.path().join("dump"), - dump_batch_size: 16, - http_addr: "127.0.0.1:7700".to_owned(), - master_key: None, - env: "development".to_owned(), - no_analytics: true, - max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), - max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), - http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), - ssl_cert_path: None, - ssl_key_path: None, - ssl_auth_path: None, - ssl_ocsp_path: None, - ssl_require_auth: false, - ssl_resumption: false, - ssl_tickets: false, - import_snapshot: None, - ignore_missing_snapshot: false, - ignore_snapshot_if_db_exists: false, - snapshot_dir: ".".into(), - schedule_snapshot: false, - snapshot_interval_sec: 0, - import_dump: None, - indexer_options: IndexerOpts::default(), - #[cfg(all(not(debug_assertions), feature = "sentry"))] - sentry_dsn: String::from(""), - #[cfg(all(not(debug_assertions), feature = "sentry"))] - no_sentry: true, - }; + let opt = default_settings(dir.path()); let data = Data::new(opt).unwrap(); let service = Service(data); - Server { service, _dir: dir } + Server { service, _dir: Some(dir) } + } + + pub async fn new_with_options(opt: Opt) -> Self { + let data = Data::new(opt).unwrap(); + let service = Service(data); + + Server { service, _dir: None } } /// Returns a view to an index. There is no guarantee that the index exists. @@ -74,3 +53,37 @@ impl Server { self.service.get("/version").await } } + +pub fn default_settings(dir: impl AsRef) -> Opt { + Opt { + db_path: dir.as_ref().join("db"), + dumps_dir: dir.as_ref().join("dump"), + dump_batch_size: 16, + http_addr: "127.0.0.1:7700".to_owned(), + master_key: None, + env: "development".to_owned(), + no_analytics: true, + max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), + max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), + http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), + ssl_cert_path: None, + ssl_key_path: None, + ssl_auth_path: None, + ssl_ocsp_path: None, + ssl_require_auth: false, + ssl_resumption: false, + ssl_tickets: false, + import_snapshot: None, + ignore_missing_snapshot: false, + ignore_snapshot_if_db_exists: false, + snapshot_dir: ".".into(), + schedule_snapshot: false, + snapshot_interval_sec: 0, + import_dump: None, + indexer_options: IndexerOpts::default(), + #[cfg(all(not(debug_assertions), feature = "sentry"))] + sentry_dsn: String::from(""), + #[cfg(all(not(debug_assertions), feature = "sentry"))] + no_sentry: true, + } +} diff --git a/meilisearch-http/tests/integration.rs b/meilisearch-http/tests/integration.rs index 8acc75ff9..b414072d4 100644 --- a/meilisearch-http/tests/integration.rs +++ b/meilisearch-http/tests/integration.rs @@ -3,8 +3,9 @@ mod documents; mod index; mod search; mod settings; -mod updates; +mod snapshot; mod stats; +mod updates; // Tests are isolated by features in different modules to allow better readability, test // targetability, and improved incremental compilation times. From 4041d9dc481b78adce546b0a68d57e0a3c8e4121 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Mar 2021 11:29:11 +0100 Subject: [PATCH 224/522] format code --- meilisearch-http/src/helpers/compression.rs | 2 +- .../src/index_controller/index_actor/actor.rs | 4 ++-- .../index_controller/index_actor/handle_impl.rs | 12 +++++------- .../src/index_controller/index_actor/message.rs | 7 ++----- .../src/index_controller/index_actor/mod.rs | 10 ++-------- .../src/index_controller/index_actor/store.rs | 10 +++++----- meilisearch-http/src/index_controller/mod.rs | 4 +++- .../src/index_controller/snapshot.rs | 11 ++++------- .../src/index_controller/update_actor/actor.rs | 8 ++++---- .../src/index_controller/update_actor/message.rs | 4 ++-- .../src/index_controller/update_actor/mod.rs | 8 ++++---- .../update_actor/update_store.rs | 16 +++++++++++----- .../src/index_controller/uuid_resolver/actor.rs | 3 +-- .../uuid_resolver/handle_impl.rs | 4 ++-- .../src/index_controller/uuid_resolver/store.rs | 4 ++-- meilisearch-http/src/routes/stats.rs | 2 +- meilisearch-http/tests/common/server.rs | 10 ++++++++-- meilisearch-http/tests/stats/mod.rs | 2 +- 18 files changed, 60 insertions(+), 61 deletions(-) diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index 201f35149..c4747cb21 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -2,7 +2,7 @@ use std::fs::{create_dir_all, File}; use std::io::Write; use std::path::Path; -use flate2::{Compression, write::GzEncoder, read::GzDecoder}; +use flate2::{read::GzDecoder, write::GzEncoder, Compression}; use tar::{Archive, Builder}; pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 89831b53a..812cf8408 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -12,11 +12,11 @@ use tokio::sync::mpsc; use tokio::task::spawn_blocking; use uuid::Uuid; +use super::{IndexError, IndexMeta, IndexMsg, IndexSettings, IndexStore, Result, UpdateResult}; use crate::index::{Document, SearchQuery, SearchResult, Settings}; use crate::index_controller::update_handler::UpdateHandler; -use crate::index_controller::{updates::Processing, UpdateMeta, get_arc_ownership_blocking}; +use crate::index_controller::{get_arc_ownership_blocking, updates::Processing, UpdateMeta}; use crate::option::IndexerOpts; -use super::{IndexSettings, Result, IndexMsg, IndexStore, IndexError, UpdateResult, IndexMeta}; pub struct IndexActor { read_receiver: Option>, diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 9c43bd6e7..dba0f9e60 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -1,12 +1,14 @@ -use std::path::{PathBuf, Path}; +use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; +use super::{ + IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore, Result, UpdateResult, +}; use crate::index::{Document, SearchQuery, SearchResult, Settings}; use crate::index_controller::IndexSettings; use crate::index_controller::{updates::Processing, UpdateMeta}; -use super::{IndexActorHandle, IndexMsg, IndexMeta, UpdateResult, Result, IndexActor, MapIndexStore}; #[derive(Clone)] pub struct IndexActorHandleImpl { @@ -102,11 +104,7 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn update_index( - &self, - uuid: Uuid, - index_settings: IndexSettings, - ) -> Result { + async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::UpdateIndex { uuid, diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 66edb5b77..46d7f6214 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -3,12 +3,9 @@ use std::path::PathBuf; use tokio::sync::oneshot; use uuid::Uuid; +use super::{IndexMeta, IndexSettings, Result, UpdateResult}; use crate::index::{Document, SearchQuery, SearchResult, Settings}; -use crate::index_controller::{ - updates::Processing, - UpdateMeta, -}; -use super::{IndexSettings, IndexMeta, UpdateResult, Result}; +use crate::index_controller::{updates::Processing, UpdateMeta}; pub enum IndexMsg { CreateIndex { diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index e3e48add7..2dc856b80 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -17,9 +17,9 @@ use crate::index_controller::{ updates::{Failed, Processed, Processing}, UpdateMeta, }; +use actor::IndexActor; use message::IndexMsg; use store::{IndexStore, MapIndexStore}; -use actor::IndexActor; pub use handle_impl::IndexActorHandleImpl; @@ -69,7 +69,6 @@ pub enum IndexError { ExistingPrimaryKey, } - #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait IndexActorHandle { @@ -97,11 +96,6 @@ pub trait IndexActorHandle { ) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; async fn get_index_meta(&self, uuid: Uuid) -> Result; - async fn update_index( - &self, - uuid: Uuid, - index_settings: IndexSettings, - ) -> Result; + async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; } - diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index a9f3cd479..6250f515e 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -1,12 +1,12 @@ -use std::path::{PathBuf, Path}; -use std::sync::Arc; use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; -use uuid::Uuid; +use heed::EnvOpenOptions; +use tokio::fs; use tokio::sync::RwLock; use tokio::task::spawn_blocking; -use tokio::fs; -use heed::EnvOpenOptions; +use uuid::Uuid; use super::{IndexError, Result}; use crate::index::Index; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 77ad957f8..3efb31f82 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -12,6 +12,7 @@ use std::time::Duration; use actix_web::web::{Bytes, Payload}; use anyhow::bail; use futures::stream::StreamExt; +use log::info; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; @@ -22,9 +23,9 @@ use crate::index::{Facets, Settings, UpdateResult}; use crate::option::Opt; use index_actor::IndexActorHandle; +use snapshot::load_snapshot; use update_actor::UpdateActorHandle; use uuid_resolver::UuidResolverHandle; -use snapshot::load_snapshot; use snapshot::SnapshotService; pub use updates::{Failed, Processed, Processing}; @@ -72,6 +73,7 @@ impl IndexController { let update_store_size = options.max_udb_size.get_bytes() as usize; if let Some(ref path) = options.import_snapshot { + info!("Loading from snapshot {:?}", path); load_snapshot( &options.db_path, path, diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 98fabcca7..5ff0b9d6f 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -39,7 +39,10 @@ where } pub async fn run(self) { - info!("Snashot scheduled every {}s.", self.snapshot_period.as_secs()); + info!( + "Snashot scheduled every {}s.", + self.snapshot_period.as_secs() + ); loop { sleep(self.snapshot_period).await; if let Err(e) = self.perform_snapshot().await { @@ -49,17 +52,11 @@ where } async fn perform_snapshot(&self) -> anyhow::Result<()> { - if !self.snapshot_path.is_file() { - bail!("Invalid snapshot file path."); - } - info!("Performing snapshot."); let temp_snapshot_dir = spawn_blocking(move || tempfile::tempdir_in(".")).await??; let temp_snapshot_path = temp_snapshot_dir.path().to_owned(); - fs::create_dir_all(&temp_snapshot_path).await?; - let uuids = self .uuid_resolver_handle .snapshot(temp_snapshot_path.clone()) diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index df949279e..688472740 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -2,15 +2,15 @@ use std::io::SeekFrom; use std::path::{Path, PathBuf}; use log::info; -use tokio::sync::mpsc; -use uuid::Uuid; use oxidized_json_checker::JsonChecker; use tokio::fs; use tokio::io::{AsyncSeekExt, AsyncWriteExt}; +use tokio::sync::mpsc; +use uuid::Uuid; -use super::{PayloadData, UpdateError, UpdateMsg, UpdateStoreStore, Result}; +use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStoreStore}; use crate::index_controller::index_actor::IndexActorHandle; -use crate::index_controller::{UpdateMeta, UpdateStatus, get_arc_ownership_blocking}; +use crate::index_controller::{get_arc_ownership_blocking, UpdateMeta, UpdateStatus}; pub struct UpdateActor { path: PathBuf, diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 628a1ebcb..8e6e3c212 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -1,9 +1,9 @@ use std::path::PathBuf; +use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use tokio::sync::{oneshot, mpsc}; -use super::{Result, PayloadData, UpdateStatus, UpdateMeta}; +use super::{PayloadData, Result, UpdateMeta, UpdateStatus}; pub enum UpdateMsg { Update { diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 1fb53a2b3..f3c3caf04 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -1,7 +1,7 @@ mod actor; -mod store; -mod message; mod handle_impl; +mod message; +mod store; mod update_store; use std::path::PathBuf; @@ -15,7 +15,7 @@ use crate::index_controller::{UpdateMeta, UpdateStatus}; use actor::UpdateActor; use message::UpdateMsg; -use store::{UpdateStoreStore, MapUpdateStoreStore}; +use store::{MapUpdateStoreStore, UpdateStoreStore}; pub use handle_impl::UpdateActorHandleImpl; @@ -51,5 +51,5 @@ pub trait UpdateActorHandle { meta: UpdateMeta, data: mpsc::Receiver>, uuid: Uuid, - ) -> Result ; + ) -> Result; } diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 63806260b..f8dcace7b 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -1,10 +1,10 @@ -use std::fs::{remove_file, create_dir_all, copy}; +use std::fs::{copy, create_dir_all, remove_file}; use std::path::{Path, PathBuf}; use std::sync::Arc; use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; -use heed::{Database, Env, EnvOpenOptions, CompactionOption}; -use parking_lot::{RwLock, Mutex}; +use heed::{CompactionOption, Database, Env, EnvOpenOptions}; +use parking_lot::{Mutex, RwLock}; use serde::{Deserialize, Serialize}; use std::fs::File; use tokio::sync::mpsc; @@ -379,7 +379,12 @@ where Ok(aborted_updates) } - pub fn snapshot(&self, txn: &mut heed::RwTxn, path: impl AsRef, uuid: Uuid) -> anyhow::Result<()> { + pub fn snapshot( + &self, + txn: &mut heed::RwTxn, + path: impl AsRef, + uuid: Uuid, + ) -> anyhow::Result<()> { let update_path = path.as_ref().join("updates"); create_dir_all(&update_path)?; @@ -389,7 +394,8 @@ where snapshot_path.push("data.mdb"); // create db snapshot - self.env.copy_to_path(&snapshot_path, CompactionOption::Enabled)?; + self.env + .copy_to_path(&snapshot_path, CompactionOption::Enabled)?; let update_files_path = update_path.join("update_files"); create_dir_all(&update_files_path)?; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 4c0a13ad7..e1abe4ddb 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -4,7 +4,7 @@ use log::{info, warn}; use tokio::sync::mpsc; use uuid::Uuid; -use super::{UuidResolveMsg, UuidStore, Result, UuidError}; +use super::{Result, UuidError, UuidResolveMsg, UuidStore}; pub struct UuidResolverActor { inbox: mpsc::Receiver, @@ -91,4 +91,3 @@ fn is_index_uid_valid(uid: &str) -> bool { uid.chars() .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') } - diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index 265ea8422..e7a4eb927 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use super::{HeedUuidStore, UuidResolverActor, UuidResolveMsg, UuidResolverHandle, Result}; +use super::{HeedUuidStore, Result, UuidResolveMsg, UuidResolverActor, UuidResolverHandle}; #[derive(Clone)] pub struct UuidResolverHandleImpl { @@ -21,7 +21,7 @@ impl UuidResolverHandleImpl { } #[async_trait::async_trait] -impl UuidResolverHandle for UuidResolverHandleImpl { +impl UuidResolverHandle for UuidResolverHandleImpl { async fn resolve(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Resolve { uid: name, ret }; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index e821f2b2b..710d48e5d 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -3,11 +3,11 @@ use std::path::{Path, PathBuf}; use heed::{ types::{ByteSlice, Str}, - Database, Env, EnvOpenOptions,CompactionOption + CompactionOption, Database, Env, EnvOpenOptions, }; use uuid::Uuid; -use super::{UUID_STORE_SIZE, UuidError, Result}; +use super::{Result, UuidError, UUID_STORE_SIZE}; #[async_trait::async_trait] pub trait UuidStore { diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index 172b5dde9..ed951149c 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -56,7 +56,7 @@ struct VersionResponse { #[get("/version", wrap = "Authentication::Private")] async fn get_version() -> HttpResponse { - HttpResponse::Ok().json(VersionResponse { + HttpResponse::Ok().json(VersionResponse { commit_sha: env!("VERGEN_SHA").to_string(), build_date: env!("VERGEN_BUILD_TIMESTAMP").to_string(), pkg_version: env!("CARGO_PKG_VERSION").to_string(), diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index da490250b..4655b10a8 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -27,14 +27,20 @@ impl Server { let data = Data::new(opt).unwrap(); let service = Service(data); - Server { service, _dir: Some(dir) } + Server { + service, + _dir: Some(dir), + } } pub async fn new_with_options(opt: Opt) -> Self { let data = Data::new(opt).unwrap(); let service = Service(data); - Server { service, _dir: None } + Server { + service, + _dir: None, + } } /// Returns a view to an index. There is no guarantee that the index exists. diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index 5e3c9db45..db04fb1c0 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -18,4 +18,4 @@ async fn test_healthyness() { let (response, status_code) = server.service.get("/health").await; assert_eq!(status_code, 200); assert_eq!(response["status"], "available"); - } +} From 83ffdc888aeab8f04d6207d49274915d4b65474c Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Mar 2021 11:30:45 +0100 Subject: [PATCH 225/522] remove bad file name test --- meilisearch-http/src/index_controller/snapshot.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 5ff0b9d6f..669df6847 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -165,21 +165,6 @@ mod test { snapshot_service.perform_snapshot().await.unwrap(); } - #[actix_rt::test] - async fn bad_file_name() { - let uuid_resolver = MockUuidResolverHandle::new(); - let update_handle = MockUpdateActorHandle::new(); - - let snapshot_service = SnapshotService::new( - uuid_resolver, - update_handle, - Duration::from_millis(100), - "directory/".into(), - ); - - assert!(snapshot_service.perform_snapshot().await.is_err()); - } - #[actix_rt::test] async fn error_performing_uuid_snapshot() { let mut uuid_resolver = MockUuidResolverHandle::new(); From d892a2643edf3669163d905e507bb1d8590c5575 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Mar 2021 11:50:52 +0100 Subject: [PATCH 226/522] fix clippy --- meilisearch-http/src/index_controller/index_actor/actor.rs | 2 +- meilisearch-http/src/index_controller/snapshot.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 812cf8408..a4228227f 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -323,7 +323,7 @@ impl IndexActor { }) .await .map_err(|e| IndexError::Error(e.into()))? - .map_err(|e| IndexError::Error(e.into()))?; + .map_err(IndexError::Error)?; } Ok(()) diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 669df6847..c433e3959 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -107,7 +107,7 @@ pub fn load_snapshot( db_path .as_ref() .canonicalize() - .unwrap_or(db_path.as_ref().to_owned()) + .unwrap_or_else(|_| db_path.as_ref().to_owned()) ) } else if !snapshot_path.as_ref().exists() && !ignore_missing_snapshot { bail!( @@ -115,7 +115,7 @@ pub fn load_snapshot( snapshot_path .as_ref() .canonicalize() - .unwrap_or(snapshot_path.as_ref().to_owned()) + .unwrap_or_else(|_| snapshot_path.as_ref().to_owned()) ) } else { Ok(()) From 79d09705d829773fde76f71cbc4e57f81fb59382 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 25 Mar 2021 09:34:29 +0100 Subject: [PATCH 227/522] perform snapshot on startup --- meilisearch-http/src/index_controller/snapshot.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index c433e3959..8eccb16bf 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -40,14 +40,14 @@ where pub async fn run(self) { info!( - "Snashot scheduled every {}s.", + "Snapshot scheduled every {}s.", self.snapshot_period.as_secs() ); loop { - sleep(self.snapshot_period).await; if let Err(e) = self.perform_snapshot().await { error!("{}", e); } + sleep(self.snapshot_period).await; } } From d029464de80a709b46d90ea684d795136f39fbd4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 25 Mar 2021 10:23:31 +0100 Subject: [PATCH 228/522] fix snapshot path --- meilisearch-http/src/index_controller/mod.rs | 4 +++ .../src/index_controller/snapshot.rs | 25 ++++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 3efb31f82..a0b119239 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -98,6 +98,10 @@ impl IndexController { update_handle.clone(), Duration::from_secs(options.snapshot_interval_sec), options.snapshot_dir.clone(), + options.db_path + .file_name() + .map(|n| n.to_owned().into_string().expect("invalid path")) + .unwrap_or_else(|| String::from("data.ms")), ); tokio::task::spawn(snapshot_service.run()); diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 8eccb16bf..6e99a5107 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -3,9 +3,9 @@ use std::time::Duration; use anyhow::bail; use log::{error, info}; -use tokio::fs; use tokio::task::spawn_blocking; use tokio::time::sleep; +use tokio::fs; use super::update_actor::UpdateActorHandle; use super::uuid_resolver::UuidResolverHandle; @@ -17,6 +17,7 @@ pub struct SnapshotService { update_handle: U, snapshot_period: Duration, snapshot_path: PathBuf, + db_name: String, } impl SnapshotService @@ -29,12 +30,14 @@ where update_handle: U, snapshot_period: Duration, snapshot_path: PathBuf, + db_name: String, ) -> Self { Self { uuid_resolver_handle, update_handle, snapshot_period, snapshot_path, + db_name, } } @@ -54,6 +57,8 @@ where async fn perform_snapshot(&self) -> anyhow::Result<()> { info!("Performing snapshot."); + fs::create_dir_all(&self.snapshot_path).await?; + let temp_snapshot_dir = spawn_blocking(move || tempfile::tempdir_in(".")).await??; let temp_snapshot_path = temp_snapshot_dir.path().to_owned(); @@ -76,18 +81,20 @@ where futures::future::try_join_all(tasks).await?; - let temp_snapshot_file = temp_snapshot_path.with_extension("temp"); - - let temp_snapshot_file_clone = temp_snapshot_file.clone(); let temp_snapshot_path_clone = temp_snapshot_path.clone(); - spawn_blocking(move || { - compression::to_tar_gz(temp_snapshot_path_clone, temp_snapshot_file_clone) + + let snapshot_dir = self.snapshot_path.clone(); + let snapshot_path = self.snapshot_path.join(format!("{}.snapshot", self.db_name)); + let snapshot_path = spawn_blocking(move || -> anyhow::Result { + let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?; + let temp_snapshot_file_path = temp_snapshot_file.path().to_owned(); + compression::to_tar_gz(temp_snapshot_path_clone, temp_snapshot_file_path)?; + temp_snapshot_file.persist(&snapshot_path)?; + Ok(snapshot_path) }) .await??; - fs::rename(temp_snapshot_file, &self.snapshot_path).await?; - - info!("Created snapshot in {:?}.", self.snapshot_path); + info!("Created snapshot in {:?}.", snapshot_path); Ok(()) } From 48507460b2778b13e667da20d7aeb3c172220404 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 25 Mar 2021 10:24:33 +0100 Subject: [PATCH 229/522] add snapshot tests --- .../src/index_controller/snapshot.rs | 26 +++++----- meilisearch-http/tests/snapshot/mod.rs | 52 +++++++++++++++++++ 2 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 meilisearch-http/tests/snapshot/mod.rs diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 6e99a5107..57bfd0940 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -57,9 +57,9 @@ where async fn perform_snapshot(&self) -> anyhow::Result<()> { info!("Performing snapshot."); - fs::create_dir_all(&self.snapshot_path).await?; - - let temp_snapshot_dir = spawn_blocking(move || tempfile::tempdir_in(".")).await??; + let snapshot_dir = self.snapshot_path.clone(); + fs::create_dir_all(&snapshot_dir).await?; + let temp_snapshot_dir = spawn_blocking(move || tempfile::tempdir_in(snapshot_dir)).await??; let temp_snapshot_path = temp_snapshot_dir.path().to_owned(); let uuids = self @@ -81,14 +81,12 @@ where futures::future::try_join_all(tasks).await?; - let temp_snapshot_path_clone = temp_snapshot_path.clone(); - let snapshot_dir = self.snapshot_path.clone(); let snapshot_path = self.snapshot_path.join(format!("{}.snapshot", self.db_name)); let snapshot_path = spawn_blocking(move || -> anyhow::Result { let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?; let temp_snapshot_file_path = temp_snapshot_file.path().to_owned(); - compression::to_tar_gz(temp_snapshot_path_clone, temp_snapshot_file_path)?; + compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?; temp_snapshot_file.persist(&snapshot_path)?; Ok(snapshot_path) }) @@ -161,12 +159,13 @@ mod test { .times(uuids_num) .returning(move |_, _| Box::pin(ok(()))); - let snapshot_path = tempfile::NamedTempFile::new_in(".").unwrap(); + let snapshot_path = tempfile::tempdir_in(".").unwrap(); let snapshot_service = SnapshotService::new( uuid_resolver, update_handle, Duration::from_millis(100), snapshot_path.path().to_owned(), + "data.ms".to_string(), ); snapshot_service.perform_snapshot().await.unwrap(); @@ -183,17 +182,18 @@ mod test { let update_handle = MockUpdateActorHandle::new(); - let snapshot_path = tempfile::NamedTempFile::new_in(".").unwrap(); + let snapshot_path = tempfile::tempdir_in(".").unwrap(); let snapshot_service = SnapshotService::new( uuid_resolver, update_handle, Duration::from_millis(100), snapshot_path.path().to_owned(), + "data.ms".to_string(), ); assert!(snapshot_service.perform_snapshot().await.is_err()); // Nothing was written to the file - assert_eq!(snapshot_path.as_file().metadata().unwrap().len(), 0); + assert!(!snapshot_path.path().join("data.ms.snapshot").exists()); } #[actix_rt::test] @@ -211,17 +211,18 @@ mod test { // abitrary error .returning(|_, _| Box::pin(err(UpdateError::UnexistingUpdate(0)))); - let snapshot_path = tempfile::NamedTempFile::new_in(".").unwrap(); + let snapshot_path = tempfile::tempdir_in(".").unwrap(); let snapshot_service = SnapshotService::new( uuid_resolver, update_handle, Duration::from_millis(100), snapshot_path.path().to_owned(), + "data.ms".to_string(), ); assert!(snapshot_service.perform_snapshot().await.is_err()); // Nothing was written to the file - assert_eq!(snapshot_path.as_file().metadata().unwrap().len(), 0); + assert!(!snapshot_path.path().join("data.ms.snapshot").exists()); } #[actix_rt::test] @@ -236,12 +237,13 @@ mod test { let update_handle = MockUpdateActorHandle::new(); - let snapshot_path = tempfile::NamedTempFile::new_in(".").unwrap(); + let snapshot_path = tempfile::tempdir_in(".").unwrap(); let snapshot_service = SnapshotService::new( uuid_resolver, update_handle, Duration::from_millis(100), snapshot_path.path().to_owned(), + "data.ms".to_string(), ); let _ = timeout(Duration::from_millis(300), snapshot_service.run()).await; diff --git a/meilisearch-http/tests/snapshot/mod.rs b/meilisearch-http/tests/snapshot/mod.rs new file mode 100644 index 000000000..727406fb3 --- /dev/null +++ b/meilisearch-http/tests/snapshot/mod.rs @@ -0,0 +1,52 @@ +use std::time::Duration; + +use crate::common::server::default_settings; +use crate::common::GetAllDocumentsOptions; +use crate::common::Server; +use tokio::time::sleep; + +use meilisearch_http::Opt; + +#[actix_rt::test] +async fn perform_snapshot() { + let temp = tempfile::tempdir_in(".").unwrap(); + let snapshot_dir = tempfile::tempdir_in(".").unwrap(); + + let options = Opt { + snapshot_dir: snapshot_dir.path().to_owned(), + snapshot_interval_sec: 1, + schedule_snapshot: true, + ..default_settings(temp.path()) + }; + + let server = Server::new_with_options(options).await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, _) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + + sleep(Duration::from_secs(2)).await; + + let temp = tempfile::tempdir_in(".").unwrap(); + + let snapshot_path = snapshot_dir + .path() + .to_owned() + .join(format!("db.snapshot")); + + let options = Opt { + import_snapshot: Some(snapshot_path), + ..default_settings(temp.path()) + }; + + let server = Server::new_with_options(options).await; + let index = server.index("test"); + + let (response_from_snapshot, _) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + + assert_eq!(response, response_from_snapshot); +} From 7d6ec7f3d37f81c710bcfc5083c2a62d461f4e9c Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 25 Mar 2021 14:21:05 +0100 Subject: [PATCH 230/522] resolve merge --- .../index_controller/uuid_resolver/actor.rs | 27 ++++++++++--------- .../uuid_resolver/handle_impl.rs | 22 +++++++-------- .../index_controller/uuid_resolver/message.rs | 13 +++++---- .../src/index_controller/uuid_resolver/mod.rs | 4 +-- .../index_controller/uuid_resolver/store.rs | 14 ++++++++++ 5 files changed, 47 insertions(+), 33 deletions(-) diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index e1abe4ddb..d5cde13e7 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -26,11 +26,8 @@ impl UuidResolverActor { Some(Create { uid: name, ret }) => { let _ = ret.send(self.handle_create(name).await); } - Some(GetOrCreate { uid: name, ret }) => { - let _ = ret.send(self.handle_get_or_create(name).await); - } - Some(Resolve { uid: name, ret }) => { - let _ = ret.send(self.handle_resolve(name).await); + Some(Get { uid: name, ret }) => { + let _ = ret.send(self.handle_get(name).await); } Some(Delete { uid: name, ret }) => { let _ = ret.send(self.handle_delete(name).await); @@ -38,6 +35,9 @@ impl UuidResolverActor { Some(List { ret }) => { let _ = ret.send(self.handle_list().await); } + Some(Insert { ret, uuid, name }) => { + let _ = ret.send(self.handle_insert(name, uuid).await); + } Some(SnapshotRequest { path, ret }) => { let _ = ret.send(self.handle_snapshot(path).await); } @@ -56,14 +56,7 @@ impl UuidResolverActor { self.store.create_uuid(uid, true).await } - async fn handle_get_or_create(&self, uid: String) -> Result { - if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)); - } - self.store.create_uuid(uid, false).await - } - - async fn handle_resolve(&self, uid: String) -> Result { + async fn handle_get(&self, uid: String) -> Result { self.store .get_uuid(uid.clone()) .await? @@ -85,6 +78,14 @@ impl UuidResolverActor { async fn handle_snapshot(&self, path: PathBuf) -> Result> { self.store.snapshot(path).await } + + async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> { + if !is_index_uid_valid(&uid) { + return Err(UuidError::BadlyFormatted(uid)); + } + self.store.insert(uid, uuid).await?; + Ok(()) + } } fn is_index_uid_valid(uid: &str) -> bool { diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index e7a4eb927..f8625b379 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -22,18 +22,9 @@ impl UuidResolverHandleImpl { #[async_trait::async_trait] impl UuidResolverHandle for UuidResolverHandleImpl { - async fn resolve(&self, name: String) -> anyhow::Result { + async fn get(&self, name: String) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Resolve { uid: name, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - - async fn get_or_create(&self, name: String) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::GetOrCreate { uid: name, ret }; + let msg = UuidResolveMsg::Get { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -67,6 +58,15 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } + async fn insert(&self, name: String, uuid: Uuid) -> anyhow::Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Insert { ret, name, uuid }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + async fn snapshot(&self, path: PathBuf) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::SnapshotRequest { path, ret }; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index 0e8323a82..975c709b3 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -4,14 +4,8 @@ use tokio::sync::oneshot; use uuid::Uuid; use super::Result; - -#[derive(Debug)] pub enum UuidResolveMsg { - Resolve { - uid: String, - ret: oneshot::Sender>, - }, - GetOrCreate { + Get { uid: String, ret: oneshot::Sender>, }, @@ -26,6 +20,11 @@ pub enum UuidResolveMsg { List { ret: oneshot::Sender>>, }, + Insert { + uuid: Uuid, + name: String, + ret: oneshot::Sender>, + }, SnapshotRequest { path: PathBuf, ret: oneshot::Sender>>, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 2361fc08b..43cd9995b 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -24,8 +24,8 @@ pub type Result = std::result::Result; #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait UuidResolverHandle { - async fn resolve(&self, name: String) -> anyhow::Result; - async fn get_or_create(&self, name: String) -> Result; + async fn get(&self, name: String) -> Result; + async fn insert(&self, name: String, uuid: Uuid) -> anyhow::Result<()>; async fn create(&self, name: String) -> anyhow::Result; async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 710d48e5d..435314911 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -17,6 +17,7 @@ pub trait UuidStore { async fn get_uuid(&self, uid: String) -> Result>; async fn delete(&self, uid: String) -> Result>; async fn list(&self) -> Result>; + async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; async fn snapshot(&self, path: PathBuf) -> Result>; } @@ -63,6 +64,7 @@ impl UuidStore for HeedUuidStore { }) .await? } + async fn get_uuid(&self, name: String) -> Result> { let env = self.env.clone(); let db = self.db; @@ -113,6 +115,18 @@ impl UuidStore for HeedUuidStore { .await? } + async fn insert(&self, name: String, uuid: Uuid) -> Result<()> { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + let mut txn = env.write_txn()?; + db.put(&mut txn, &name, uuid.as_bytes())?; + txn.commit()?; + Ok(()) + }) + .await? + } + async fn snapshot(&self, mut path: PathBuf) -> Result> { let env = self.env.clone(); let db = self.db; From d7c077cffbbd046ba70b5d7f71954aebaa0f73ff Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 25 Mar 2021 14:48:51 +0100 Subject: [PATCH 231/522] atomic snapshot import --- .../src/index_controller/snapshot.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 57bfd0940..b494d2481 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -3,9 +3,9 @@ use std::time::Duration; use anyhow::bail; use log::{error, info}; +use tokio::fs; use tokio::task::spawn_blocking; use tokio::time::sleep; -use tokio::fs; use super::update_actor::UpdateActorHandle; use super::uuid_resolver::UuidResolverHandle; @@ -59,7 +59,8 @@ where let snapshot_dir = self.snapshot_path.clone(); fs::create_dir_all(&snapshot_dir).await?; - let temp_snapshot_dir = spawn_blocking(move || tempfile::tempdir_in(snapshot_dir)).await??; + let temp_snapshot_dir = + spawn_blocking(move || tempfile::tempdir_in(snapshot_dir)).await??; let temp_snapshot_path = temp_snapshot_dir.path().to_owned(); let uuids = self @@ -82,7 +83,9 @@ where futures::future::try_join_all(tasks).await?; let snapshot_dir = self.snapshot_path.clone(); - let snapshot_path = self.snapshot_path.join(format!("{}.snapshot", self.db_name)); + let snapshot_path = self + .snapshot_path + .join(format!("{}.snapshot", self.db_name)); let snapshot_path = spawn_blocking(move || -> anyhow::Result { let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?; let temp_snapshot_file_path = temp_snapshot_file.path().to_owned(); @@ -105,7 +108,14 @@ pub fn load_snapshot( ignore_missing_snapshot: bool, ) -> anyhow::Result<()> { if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() { - compression::from_tar_gz(snapshot_path, db_path) + match compression::from_tar_gz(snapshot_path, &db_path) { + Ok(()) => Ok(()), + Err(e) => { + // clean created db folder + std::fs::remove_dir_all(&db_path)?; + Err(e) + } + } } else if db_path.as_ref().exists() && !ignore_snapshot_if_db_exists { bail!( "database already exists at {:?}, try to delete it or rename it", From b9f79c8df089535b34209dd61677e45a432468ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Fri, 26 Mar 2021 12:09:51 +0100 Subject: [PATCH 232/522] Update display --- meilisearch-http/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 9415f52d5..b04c556db 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -114,7 +114,7 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { eprintln!("{}", ascii_name); eprintln!("Database path:\t\t{:?}", opt.db_path); - eprintln!("Server listening on:\t{:?}", opt.http_addr); + eprintln!("Server listening on:\t\"http://{}\"", opt.http_addr); eprintln!("Environment:\t\t{:?}", opt.env); eprintln!("Commit SHA:\t\t{:?}", env!("VERGEN_SHA").to_string()); eprintln!( @@ -137,7 +137,7 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { ); eprintln!( - "Amplitude Analytics:\t{:?}", + "Anonymous telemetry:\t{:?}", if !opt.no_analytics { "Enabled" } else { From 0f2143e7fd1f040b56db08bf5e19ff12b279a788 Mon Sep 17 00:00:00 2001 From: tamo Date: Fri, 26 Mar 2021 14:15:12 +0100 Subject: [PATCH 233/522] remove the now useless dead_code flags --- meilisearch-http/src/index/search.rs | 1 - meilisearch-http/src/index_controller/snapshot.rs | 1 - meilisearch-http/tests/common/index.rs | 1 - 3 files changed, 3 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 2002266a1..7311687d2 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -20,7 +20,6 @@ const fn default_search_limit() -> usize { #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -#[allow(dead_code)] pub struct SearchQuery { pub q: Option, pub offset: Option, diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index b494d2481..8557fe04e 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -11,7 +11,6 @@ use super::update_actor::UpdateActorHandle; use super::uuid_resolver::UuidResolverHandle; use crate::helpers::compression; -#[allow(dead_code)] pub struct SnapshotService { uuid_resolver_handle: R, update_handle: U, diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 8fda99ef9..a7a9a5378 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -96,7 +96,6 @@ impl Index<'_> { self.service.get(url).await } - #[allow(dead_code)] pub async fn list_updates(&self) -> (Value, StatusCode) { let url = format!("/indexes/{}/updates", self.uid); self.service.get(url).await From 5bc464dc53db14e222a955b2e9ae1960089b4d84 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 30 Mar 2021 15:01:51 +0300 Subject: [PATCH 234/522] chore(ci): cache dependencies in Docker build --- Dockerfile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9898d02db..50ddddfcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,10 +9,23 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y WORKDIR /meilisearch -COPY . . +COPY Cargo.lock . +COPY Cargo.toml . + +COPY meilisearch-error/Cargo.toml meilisearch-error/ +COPY meilisearch-http/Cargo.lock meilisearch-http/ +COPY meilisearch-http/Cargo.toml meilisearch-http/ ENV RUSTFLAGS="-C target-feature=-crt-static" +# Create dummy main.rs files for each workspace member to be able to compile all the dependencies +RUN find . -type d -name "meilisearch-*" | xargs -I{} sh -c 'mkdir {}/src; echo "fn main() { }" > {}/src/main.rs;' +# Use `cargo build` instead of `cargo vendor` because we need to not only download but compile dependencies too +RUN $HOME/.cargo/bin/cargo build --release +# Cleanup dummy main.rs files +RUN find . -path "*/src/main.rs" -delete + +COPY . . RUN $HOME/.cargo/bin/cargo build --release # Run From 3d51db59292d96f27b8360b229e4416413a29e3e Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 30 Mar 2021 20:03:13 +0300 Subject: [PATCH 235/522] fix(ci, http): commit_sha and commit_date in docker builds chore(ci): cache dependencies in Docker build --- .github/workflows/publish_to_docker.yml | 6 ++++++ Dockerfile | 5 +++++ meilisearch-http/src/routes/stats.rs | 15 ++++++++++++--- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_to_docker.yml b/.github/workflows/publish_to_docker.yml index d724f7253..60422a88e 100644 --- a/.github/workflows/publish_to_docker.yml +++ b/.github/workflows/publish_to_docker.yml @@ -11,10 +11,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 + - name: Set COMMIT_DATE env variable + run: | + echo "COMMIT_DATE=$( git log --pretty=format:'%ad' -n1 --date=short )" >> $GITHUB_ENV - name: Publish to Registry uses: elgohr/Publish-Docker-Github-Action@master + env: + COMMIT_SHA: ${{ github.sha }} with: name: getmeili/meilisearch username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} tag_names: true + buildargs: COMMIT_SHA,COMMIT_DATE diff --git a/Dockerfile b/Dockerfile index 50ddddfcc..a45d2f7e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,11 @@ RUN $HOME/.cargo/bin/cargo build --release # Cleanup dummy main.rs files RUN find . -path "*/src/main.rs" -delete +ARG COMMIT_SHA +ARG COMMIT_DATE +ENV COMMIT_SHA=${COMMIT_SHA} +ENV COMMIT_DATE=${COMMIT_DATE} + COPY . . RUN $HOME/.cargo/bin/cargo build --release diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index 172b5dde9..108c67ca9 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -56,9 +56,18 @@ struct VersionResponse { #[get("/version", wrap = "Authentication::Private")] async fn get_version() -> HttpResponse { - HttpResponse::Ok().json(VersionResponse { - commit_sha: env!("VERGEN_SHA").to_string(), - build_date: env!("VERGEN_BUILD_TIMESTAMP").to_string(), + let commit_sha = match option_env!("COMMIT_SHA") { + Some("") | None => env!("VERGEN_SHA"), + Some(commit_sha) => commit_sha + }; + let commit_date = match option_env!("COMMIT_DATE") { + Some("") | None => env!("VERGEN_COMMIT_DATE"), + Some(commit_date) => commit_date + }; + + HttpResponse::Ok().json(VersionResponse { + commit_sha: commit_sha.to_string(), + build_date: commit_date.to_string(), pkg_version: env!("CARGO_PKG_VERSION").to_string(), }) } From d8af4a72025299ce020a422542386c1b680de010 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 31 Mar 2021 20:07:40 +0200 Subject: [PATCH 236/522] ignore snapshot test (#130) --- meilisearch-http/tests/snapshot/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/meilisearch-http/tests/snapshot/mod.rs b/meilisearch-http/tests/snapshot/mod.rs index 727406fb3..36763a1ab 100644 --- a/meilisearch-http/tests/snapshot/mod.rs +++ b/meilisearch-http/tests/snapshot/mod.rs @@ -7,6 +7,7 @@ use tokio::time::sleep; use meilisearch_http::Opt; +#[ignore] #[actix_rt::test] async fn perform_snapshot() { let temp = tempfile::tempdir_in(".").unwrap(); From 96cffeab1ec349ffb63d6f2adc8b8af936db9929 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Wed, 17 Mar 2021 13:54:17 +0100 Subject: [PATCH 237/522] update all the response format to be ISO with meilisearch, see #64 --- meilisearch-http/src/routes/document.rs | 24 ++++++++++---- meilisearch-http/tests/common/index.rs | 2 +- .../tests/documents/add_documents.rs | 31 ++++++++++--------- .../tests/documents/delete_documents.rs | 12 +++---- .../tests/documents/get_documents.rs | 2 +- 5 files changed, 43 insertions(+), 28 deletions(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 8e337d275..7b2bb4628 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -84,7 +84,9 @@ async fn delete_document( .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]) .await { - Ok(result) => Ok(HttpResponse::Ok().json(result)), + Ok(update_status) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -119,7 +121,7 @@ async fn get_all_documents( ) .await { - Ok(docs) => Ok(HttpResponse::Ok().json(docs)), + Ok(documents) => Ok(HttpResponse::Ok().json(documents)), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -133,6 +135,7 @@ struct UpdateDocumentsQuery { } /// Route used when the payload type is "application/json" +/// Used to add or replace documents #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn add_documents( data: web::Data, @@ -151,7 +154,9 @@ async fn add_documents( .await; match addition_result { - Ok(update) => Ok(HttpResponse::Ok().json(update)), + Ok(update_status) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -200,7 +205,9 @@ async fn update_documents( .await; match addition_result { - Ok(update) => Ok(HttpResponse::Ok().json(update)), + Ok(update) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -226,20 +233,25 @@ async fn delete_documents( .collect(); match data.delete_documents(path.index_uid.clone(), ids).await { - Ok(result) => Ok(HttpResponse::Ok().json(result)), + Ok(update_status) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } } } +/// delete all documents #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn clear_all_documents( data: web::Data, path: web::Path, ) -> Result { match data.clear_documents(path.index_uid.clone()).await { - Ok(update) => Ok(HttpResponse::Ok().json(update)), + Ok(update_status) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 8fda99ef9..b08381ee7 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -23,7 +23,7 @@ impl Index<'_> { .service .post_str(url, include_str!("../assets/test_set.json")) .await; - assert_eq!(code, 200); + assert_eq!(code, 202); let update_id = response["updateId"].as_i64().unwrap(); self.wait_update_id(update_id as u64).await; update_id as u64 diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index dd10541b5..87e5cecb7 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -16,13 +16,16 @@ async fn add_documents_no_index_creation() { ]); let (response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 200); - assert_eq!(response["status"], "pending"); + assert_eq!(code, 202); assert_eq!(response["updateId"], 0); - assert_eq!(response["meta"]["type"], "DocumentsAddition"); - assert_eq!(response["meta"]["format"], "Json"); - assert_eq!(response["meta"]["primaryKey"], Value::Null); - assert!(response.get("enqueuedAt").is_some()); + /* + * currently we don’t check these field to stay ISO with meilisearch + * assert_eq!(response["status"], "pending"); + * assert_eq!(response["meta"]["type"], "DocumentsAddition"); + * assert_eq!(response["meta"]["format"], "Json"); + * assert_eq!(response["meta"]["primaryKey"], Value::Null); + * assert!(response.get("enqueuedAt").is_some()); + */ index.wait_update_id(0).await; @@ -75,7 +78,7 @@ async fn document_addition_with_primary_key() { } ]); let (_response, code) = index.add_documents(documents, Some("primary")).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(0).await; @@ -102,7 +105,7 @@ async fn document_update_with_primary_key() { } ]); let (_response, code) = index.update_documents(documents, Some("primary")).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(0).await; @@ -131,7 +134,7 @@ async fn add_documents_with_primary_key_and_primary_key_already_exists() { ]); let (_response, code) = index.add_documents(documents, Some("id")).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(0).await; @@ -160,7 +163,7 @@ async fn update_documents_with_primary_key_and_primary_key_already_exists() { ]); let (_response, code) = index.update_documents(documents, Some("id")).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(0).await; let (response, code) = index.get_update(0).await; @@ -187,7 +190,7 @@ async fn replace_document() { ]); let (_response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(0).await; @@ -199,7 +202,7 @@ async fn replace_document() { ]); let (_response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(1).await; @@ -246,7 +249,7 @@ async fn update_document() { ]); let (_response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(0).await; @@ -258,7 +261,7 @@ async fn update_document() { ]); let (_response, code) = index.update_documents(documents, None).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(1).await; diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs index e7a01acd4..b69b4c11f 100644 --- a/meilisearch-http/tests/documents/delete_documents.rs +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -15,7 +15,7 @@ async fn delete_one_unexisting_document() { let index = server.index("test"); index.create(None).await; let (_response, code) = index.delete_document(0).await; - assert_eq!(code, 200); + assert_eq!(code, 202); let update = index.wait_update_id(0).await; assert_eq!(update["status"], "processed"); } @@ -29,7 +29,7 @@ async fn delete_one_document() { .await; index.wait_update_id(0).await; let (_response, code) = server.index("test").delete_document(0).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(1).await; let (_response, code) = index.get_document(0, None).await; @@ -55,7 +55,7 @@ async fn clear_all_documents() { .await; index.wait_update_id(0).await; let (_response, code) = index.clear_all_documents().await; - assert_eq!(code, 200); + assert_eq!(code, 202); let _update = index.wait_update_id(1).await; let (response, code) = index @@ -72,7 +72,7 @@ async fn clear_all_documents_empty_index() { index.create(None).await; let (_response, code) = index.clear_all_documents().await; - assert_eq!(code, 200); + assert_eq!(code, 202); let _update = index.wait_update_id(0).await; let (response, code) = index @@ -96,7 +96,7 @@ async fn delete_batch() { index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; index.wait_update_id(0).await; let (_response, code) = index.delete_batch(vec![1, 0]).await; - assert_eq!(code, 200); + assert_eq!(code, 202); let _update = index.wait_update_id(1).await; let (response, code) = index @@ -114,7 +114,7 @@ async fn delete_no_document_batch() { index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; index.wait_update_id(0).await; let (_response, code) = index.delete_batch(vec![]).await; - assert_eq!(code, 200); + assert_eq!(code, 202); let _update = index.wait_update_id(1).await; let (response, code) = index diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index d54234860..73e29576c 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -33,7 +33,7 @@ async fn get_document() { } ]); let (_, code) = index.add_documents(documents, None).await; - assert_eq!(code, 200); + assert_eq!(code, 202); index.wait_update_id(0).await; let (response, code) = index.get_document(0, None).await; assert_eq!(code, 200); From 79c63049d7fb67ebf8a85da97946232aabe4e25c Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 29 Mar 2021 20:19:37 +0200 Subject: [PATCH 238/522] update the settings routes --- meilisearch-http/src/index/updates.rs | 3 +++ meilisearch-http/src/routes/settings/mod.rs | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index a519eaa82..79558dd92 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -43,6 +43,9 @@ pub struct Settings { skip_serializing_if = "Option::is_none" )] pub ranking_rules: Option>>, + + // TODO we are missing the stopWords, synonyms and distinctAttribute for the GET settings + // request } impl Settings { diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index ee9b3e325..b5989bf87 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -131,7 +131,9 @@ async fn update_all( .update_settings(index_uid.into_inner(), body.into_inner(), true) .await { - Ok(update_result) => Ok(HttpResponse::Accepted().json(update_result)), + Ok(update_result) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_result.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -161,7 +163,9 @@ async fn delete_all( .update_settings(index_uid.into_inner(), settings, false) .await { - Ok(update_result) => Ok(HttpResponse::Accepted().json(update_result)), + Ok(update_result) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_result.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } From 73973e2b9ef2f93aabc9dfc22fa787b187bfd058 Mon Sep 17 00:00:00 2001 From: tamo Date: Thu, 1 Apr 2021 15:50:45 +0200 Subject: [PATCH 239/522] fix more settings routes --- meilisearch-http/src/routes/settings/mod.rs | 8 ++++++-- meilisearch-http/tests/settings/get_settings.rs | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index b5989bf87..8c6e04b84 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -27,7 +27,9 @@ macro_rules! make_setting_route { ..Default::default() }; match data.update_settings(index_uid.into_inner(), settings, false).await { - Ok(update_status) => Ok(HttpResponse::Ok().json(update_status)), + Ok(update_status) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -46,7 +48,9 @@ macro_rules! make_setting_route { }; match data.update_settings(index_uid.into_inner(), settings, true).await { - Ok(update_status) => Ok(HttpResponse::Ok().json(update_status)), + Ok(update_status) => { + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + } Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index b1030aea9..d234cbb2b 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -142,7 +142,7 @@ macro_rules! test_setting_routes { .map(|c| if c == '_' { '-' } else { c }) .collect::()); let (response, code) = server.service.post(url, serde_json::Value::Null).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 202, "{}", response); let (response, code) = server.index("test").get().await; assert_eq!(code, 200, "{}", response); } From 4ee6ce787180514cacd8ba8e69be37079a8f0951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 1 Apr 2021 17:16:16 +0200 Subject: [PATCH 240/522] Next release --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afbb6eb0c..94d2d8ba1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1813,7 +1813,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.21.0-alpha.1" +version = "0.21.0-alpha.2" dependencies = [ "actix-cors", "actix-http 3.0.0-beta.4", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 93776269f..a7564f4d9 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "MeiliSearch HTTP server" edition = "2018" license = "MIT" name = "meilisearch-http" -version = "0.21.0-alpha.1" +version = "0.21.0-alpha.2" [[bin]] name = "meilisearch" path = "src/main.rs" From 40ef9a3c6a9ce73532a13c2419c6a4fa2d0a55e2 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 6 Apr 2021 15:41:03 +0200 Subject: [PATCH 241/522] push a first implementation of the stop_words --- Cargo.lock | 30 +++++++++++++++---- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/index/mod.rs | 15 +++++++++- meilisearch-http/src/index/updates.rs | 19 ++++++++++-- meilisearch-http/src/routes/settings/mod.rs | 9 +++++- .../tests/settings/get_settings.rs | 6 ++-- 6 files changed, 66 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94d2d8ba1..7fd30f444 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" version = "0.3.0" @@ -1845,7 +1847,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.1.1 (git+https://github.com/meilisearch/Tokenizer.git?branch=main)", "memmap", "milli", "mime", @@ -1875,6 +1877,22 @@ dependencies = [ "vergen", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.1.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.0#833c48b2ee39071f8b4f51abd15122afdb3c8c06" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "meilisearch-tokenizer" version = "0.1.1" @@ -1919,7 +1937,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?rev=b7b23cd#b7b23cd4a8e62932c66c2ebedf9d89ddf089e299" +source = "git+https://github.com/meilisearch/milli.git?rev=2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c#2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c" dependencies = [ "anyhow", "bstr", @@ -1939,7 +1957,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.1.1 (git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.0)", "memmap", "num-traits", "obkv", @@ -2234,8 +2252,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pest" version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" dependencies = [ "ucd-trie", ] @@ -2243,7 +2260,8 @@ dependencies = [ [[package]] name = "pest" version = "2.1.3" -source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" dependencies = [ "ucd-trie", ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index a7564f4d9..03ae35729 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -42,7 +42,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "b7b23cd" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c" } mime = "0.3.16" once_cell = "1.5.2" parking_lot = "0.11.1" diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 188afd522..dfd2ebdc4 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,7 +1,7 @@ mod search; mod updates; -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; use std::ops::Deref; use std::sync::Arc; @@ -51,11 +51,24 @@ impl Index { .map(|c| c.to_string()) .collect(); + let stop_words = self + .stop_words(&txn)? + .map(|stop_words| -> anyhow::Result> { + Ok(stop_words + .stream() + .into_strs()? + .into_iter() + .collect()) + }) + .transpose()? + .unwrap_or_else(BTreeSet::new); + Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), searchable_attributes: Some(Some(searchable_attributes)), attributes_for_faceting: Some(Some(faceted_attributes)), ranking_rules: Some(Some(criteria)), + stop_words: Some(Some(stop_words)), }) } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 79558dd92..085115af6 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::io; use std::num::NonZeroUsize; @@ -44,8 +44,12 @@ pub struct Settings { )] pub ranking_rules: Option>>, - // TODO we are missing the stopWords, synonyms and distinctAttribute for the GET settings - // request + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] + pub stop_words: Option>>, } impl Settings { @@ -55,6 +59,7 @@ impl Settings { searchable_attributes: Some(None), attributes_for_faceting: Some(None), ranking_rules: Some(None), + stop_words: Some(None), } } } @@ -170,6 +175,14 @@ impl Index { } } + // We transpose the settings JSON struct into a real setting update. + if let Some(ref stop_words) = settings.stop_words { + match stop_words { + Some(stop_words) => builder.set_stop_words(stop_words.clone()), + _ => builder.reset_stop_words(), + } + } + let result = builder .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 8c6e04b84..732888ec2 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -91,6 +91,12 @@ make_setting_route!( searchable_attributes ); +make_setting_route!( + "/indexes/{index_uid}/settings/stop-words", + std::collections::BTreeSet, + stop_words +); + //make_setting_route!( //"/indexes/{index_uid}/settings/distinct-attribute", //String, @@ -122,7 +128,8 @@ macro_rules! create_services { create_services!( attributes_for_faceting, displayed_attributes, - searchable_attributes + searchable_attributes, + stop_words ); #[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index d234cbb2b..82554ee22 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -16,21 +16,21 @@ async fn get_settings() { let (response, code) = index.settings().await; assert_eq!(code, 200); let settings = response.as_object().unwrap(); - assert_eq!(settings.keys().len(), 4); + assert_eq!(settings.keys().len(), 5); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["attributesForFaceting"], json!({})); assert_eq!( settings["rankingRules"], json!([ - "typo", "words", + "typo", "proximity", "attribute", - "wordsPosition", "exactness" ]) ); + assert_eq!(settings["stopWords"], json!(null)); } #[actix_rt::test] From b1962c8e029d06ac14f57abdf64214646530aa18 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 6 Apr 2021 15:43:27 +0200 Subject: [PATCH 242/522] remove legacy files from meilisearch that have been replaced by a macro in routes/settings/mod.rs --- .../settings/attributes_for_faceting.rs | 43 ------------------- .../routes/settings/displayed_attributes.rs | 25 ----------- .../routes/settings/searchable_attributes.rs | 34 --------------- .../src/routes/settings/stop_words.rs | 33 -------------- .../tests/settings/get_settings.rs | 2 +- 5 files changed, 1 insertion(+), 136 deletions(-) delete mode 100644 meilisearch-http/src/routes/settings/attributes_for_faceting.rs delete mode 100644 meilisearch-http/src/routes/settings/displayed_attributes.rs delete mode 100644 meilisearch-http/src/routes/settings/searchable_attributes.rs delete mode 100644 meilisearch-http/src/routes/settings/stop_words.rs diff --git a/meilisearch-http/src/routes/settings/attributes_for_faceting.rs b/meilisearch-http/src/routes/settings/attributes_for_faceting.rs deleted file mode 100644 index 6c881cff3..000000000 --- a/meilisearch-http/src/routes/settings/attributes_for_faceting.rs +++ /dev/null @@ -1,43 +0,0 @@ -use actix_web::{web, HttpResponse, get}; - -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::make_update_delete_routes; -use crate::Data; - -#[get( - "/indexes/{index_uid}/settings/attributes-for-faceting", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - index_uid: web::Path, -) -> Result { - let index = data - .db - .load() - .open_index(&index_uid.as_ref()) - .ok_or(Error::index_not_found(&index_uid.as_ref()))?; - - let attributes_for_faceting = data.db.load().main_read::<_, _, ResponseError>(|reader| { - let schema = index.main.schema(reader)?; - let attrs = index.main.attributes_for_faceting(reader)?; - let attr_names = match (&schema, &attrs) { - (Some(schema), Some(attrs)) => attrs - .iter() - .filter_map(|&id| schema.name(id)) - .map(str::to_string) - .collect(), - _ => vec![], - }; - Ok(attr_names) - })?; - - Ok(HttpResponse::Ok().json(attributes_for_faceting)) -} - -make_update_delete_routes!( - "/indexes/{index_uid}/settings/attributes-for-faceting", - Vec, - attributes_for_faceting -); diff --git a/meilisearch-http/src/routes/settings/displayed_attributes.rs b/meilisearch-http/src/routes/settings/displayed_attributes.rs deleted file mode 100644 index b9f36f718..000000000 --- a/meilisearch-http/src/routes/settings/displayed_attributes.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::collections::HashSet; - -use actix_web::{web, HttpResponse, get}; - -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::make_update_delete_routes; -use crate::Data; - -#[get( - "/indexes/{index_uid}/settings/displayed-attributes", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - index_uid: web::Path, -) -> Result { - todo!() -} - -make_update_delete_routes!( - "/indexes/{index_uid}/settings/displayed-attributes", - HashSet, - displayed_attributes -); diff --git a/meilisearch-http/src/routes/settings/searchable_attributes.rs b/meilisearch-http/src/routes/settings/searchable_attributes.rs deleted file mode 100644 index a337b0435..000000000 --- a/meilisearch-http/src/routes/settings/searchable_attributes.rs +++ /dev/null @@ -1,34 +0,0 @@ -use actix_web::{web, HttpResponse, get}; - -use crate::data::get_indexed_attributes; -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::make_update_delete_routes; -use crate::Data; - -#[get( - "/indexes/{index_uid}/settings/searchable-attributes", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - index_uid: web::Path, -) -> Result { - let index = data - .db - .load() - .open_index(&index_uid.as_ref()) - - .ok_or(Error::index_not_found(&index_uid.as_ref()))?; - let reader = data.db.load().main_read_txn()?; - let schema = index.main.schema(&reader)?; - let searchable_attributes: Option> = schema.as_ref().map(get_indexed_attributes); - - Ok(HttpResponse::Ok().json(searchable_attributes)) -} - -make_update_delete_routes!( - "/indexes/{index_uid}/settings/searchable-attributes", - Vec, - searchable_attributes -); diff --git a/meilisearch-http/src/routes/settings/stop_words.rs b/meilisearch-http/src/routes/settings/stop_words.rs deleted file mode 100644 index 05a753f46..000000000 --- a/meilisearch-http/src/routes/settings/stop_words.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::collections::BTreeSet; - -use crate::make_update_delete_routes; -use actix_web::{web, HttpResponse, get}; - -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::Data; - -#[get( - "/indexes/{index_uid}/settings/stop-words", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - index_uid: web::Path, -) -> Result { - let index = data - .db - .load() - .open_index(&index_uid.as_ref()) - .ok_or(Error::index_not_found(&index_uid.as_ref()))?; - let reader = data.db.load().main_read_txn()?; - let stop_words = index.main.stop_words(&reader)?; - - Ok(HttpResponse::Ok().json(stop_words)) -} - -make_update_delete_routes!( - "/indexes/{index_uid}/settings/stop-words", - BTreeSet, - stop_words -); diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 82554ee22..34ab45be1 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -30,7 +30,7 @@ async fn get_settings() { "exactness" ]) ); - assert_eq!(settings["stopWords"], json!(null)); + assert_eq!(settings["stopWords"], json!([])); } #[actix_rt::test] From dcd60a5b459c5c17346bb0698bdbc8c30cd14357 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 6 Apr 2021 18:29:38 +0200 Subject: [PATCH 243/522] add more tests for the stop_words --- meilisearch-http/tests/settings/get_settings.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 34ab45be1..be09a5090 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -78,13 +78,14 @@ async fn reset_all_settings() { let server = Server::new().await; let index = server.index("test"); index - .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"]})) + .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"], "stopWords": ["the"] })) .await; index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); assert_eq!(response["displayedAttributes"], json!(["foo"])); assert_eq!(response["searchableAttributes"], json!(["bar"])); + assert_eq!(response["stopWords"], json!(["the"])); index.delete_settings().await; index.wait_update_id(1).await; @@ -93,6 +94,7 @@ async fn reset_all_settings() { assert_eq!(code, 200); assert_eq!(response["displayedAttributes"], json!(["*"])); assert_eq!(response["searchableAttributes"], json!(["*"])); + assert_eq!(response["stopWords"], json!([])); } #[actix_rt::test] @@ -166,5 +168,6 @@ macro_rules! test_setting_routes { test_setting_routes!( attributes_for_faceting, displayed_attributes, - searchable_attributes + searchable_attributes, + stop_words ); From cb23775d180c69f18a91619ffc8bf4cc6406f85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 7 Apr 2021 19:46:36 +0200 Subject: [PATCH 244/522] Rename pending into enqueued --- .../src/index_controller/update_actor/actor.rs | 2 +- .../update_actor/update_store.rs | 8 ++++---- .../src/index_controller/updates.rs | 18 +++++++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 688472740..d87b910d6 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -145,7 +145,7 @@ where // The payload is valid, we can register it to the update store. update_store .register_update(meta, path, uuid) - .map(UpdateStatus::Pending) + .map(UpdateStatus::Enqueued) .map_err(|e| UpdateError::Error(Box::new(e))) }) .await diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index f8dcace7b..a083eb186 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -17,7 +17,7 @@ type BEU64 = heed::zerocopy::U64; #[derive(Clone)] pub struct UpdateStore { pub env: Env, - pending_meta: Database, SerdeJson>>, + pending_meta: Database, SerdeJson>>, pending: Database, SerdeJson>, processed_meta: Database, SerdeJson>>, failed_meta: Database, SerdeJson>>, @@ -167,7 +167,7 @@ where meta: M, content: impl AsRef, index_uuid: Uuid, - ) -> heed::Result> { + ) -> heed::Result> { let mut wtxn = self.env.write_txn()?; // We ask the update store to give us a new update id, this is safe, @@ -177,7 +177,7 @@ where let update_id = self.new_update_id(&wtxn)?; let update_key = BEU64::new(update_id); - let meta = Pending::new(meta, update_id, index_uuid); + let meta = Enqueued::new(meta, update_id, index_uuid); self.pending_meta.put(&mut wtxn, &update_key, &meta)?; self.pending .put(&mut wtxn, &update_key, &content.as_ref().to_owned())?; @@ -303,7 +303,7 @@ where } if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Pending(meta))); + return Ok(Some(UpdateStatus::Enqueued(meta))); } if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 3c70aee71..42712a396 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -4,14 +4,14 @@ use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] -pub struct Pending { +pub struct Enqueued { pub update_id: u64, pub meta: M, pub enqueued_at: DateTime, pub index_uuid: Uuid, } -impl Pending { +impl Enqueued { pub fn new(meta: M, update_id: u64, index_uuid: Uuid) -> Self { Self { enqueued_at: Utc::now(), @@ -63,7 +63,7 @@ impl Processed { #[serde(rename_all = "camelCase")] pub struct Processing { #[serde(flatten)] - pub from: Pending, + pub from: Enqueued, pub started_processing_at: DateTime, } @@ -101,7 +101,7 @@ impl Processing { #[serde(rename_all = "camelCase")] pub struct Aborted { #[serde(flatten)] - from: Pending, + from: Enqueued, aborted_at: DateTime, } @@ -130,7 +130,7 @@ impl Failed { #[serde(tag = "status", rename_all = "camelCase")] pub enum UpdateStatus { Processing(Processing), - Pending(Pending), + Enqueued(Enqueued), Processed(Processed), Aborted(Aborted), Failed(Failed), @@ -140,7 +140,7 @@ impl UpdateStatus { pub fn id(&self) -> u64 { match self { UpdateStatus::Processing(u) => u.id(), - UpdateStatus::Pending(u) => u.id(), + UpdateStatus::Enqueued(u) => u.id(), UpdateStatus::Processed(u) => u.id(), UpdateStatus::Aborted(u) => u.id(), UpdateStatus::Failed(u) => u.id(), @@ -155,9 +155,9 @@ impl UpdateStatus { } } -impl From> for UpdateStatus { - fn from(other: Pending) -> Self { - Self::Pending(other) +impl From> for UpdateStatus { + fn from(other: Enqueued) -> Self { + Self::Enqueued(other) } } From 51ba1bd7d35029f1495950f0adc38bd6720ef84b Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 1 Apr 2021 17:49:11 +0300 Subject: [PATCH 245/522] fix(http, index): init analyzer with optional stop words Next release update tokenizer --- Cargo.lock | 28 ++++++---------------------- meilisearch-http/Cargo.toml | 22 +++++++++++----------- meilisearch-http/src/index/search.rs | 5 ++++- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7fd30f444..461a4789b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1847,7 +1847,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer 0.1.1 (git+https://github.com/meilisearch/Tokenizer.git?branch=main)", + "meilisearch-tokenizer", "memmap", "milli", "mime", @@ -1893,22 +1893,6 @@ dependencies = [ "whatlang", ] -[[package]] -name = "meilisearch-tokenizer" -version = "0.1.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#31ba3ff4a15501f12b7d37ac64ddce7c35a9757c" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - [[package]] name = "memchr" version = "2.3.4" @@ -1937,7 +1921,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?rev=2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c#2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.1.0#2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c" dependencies = [ "anyhow", "bstr", @@ -1957,7 +1941,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer 0.1.1 (git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.0)", + "meilisearch-tokenizer", "memmap", "num-traits", "obkv", @@ -2252,7 +2236,8 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pest" version = "2.1.3" -source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" dependencies = [ "ucd-trie", ] @@ -2260,8 +2245,7 @@ dependencies = [ [[package]] name = "pest" version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" dependencies = [ "ucd-trie", ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 03ae35729..3e04b876a 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -35,14 +35,14 @@ futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } heed = "0.10.6" http = "0.2.1" -indexmap = { version = "1.3.2", features = ["serde-1"] } +indexmap = { version = "1.3.2", features = ["serde-1"] } itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.0" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.0" } mime = "0.3.16" once_cell = "1.5.2" parking_lot = "0.11.1" @@ -66,14 +66,14 @@ oxidized-json-checker = "0.3.2" [dependencies.sentry] default-features = false features = [ - "with_client_implementation", - "with_panic", - "with_failure", - "with_device_info", - "with_rust_info", - "with_reqwest_transport", - "with_rustls", - "with_env_logger" + "with_client_implementation", + "with_panic", + "with_failure", + "with_device_info", + "with_rust_info", + "with_reqwest_transport", + "with_rustls", + "with_env_logger" ] optional = true version = "0.18.1" diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 7311687d2..4b9753b82 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -155,7 +155,10 @@ pub struct Highlighter<'a, A> { impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { pub fn new(stop_words: &'a fst::Set) -> Self { - let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); + let mut config = AnalyzerConfig::default(); + config.stop_words(stop_words); + + let analyzer = Analyzer::new(config); Self { analyzer } } From dd9eae8c26816b94df8095276783795f1ff8429d Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 1 Apr 2021 17:44:42 +0300 Subject: [PATCH 246/522] feat(http): stats route --- Cargo.lock | 37 ++++++++++++ meilisearch-http/Cargo.toml | 1 + meilisearch-http/src/data/mod.rs | 6 +- meilisearch-http/src/index/mod.rs | 16 ++++- .../src/index_controller/index_actor/actor.rs | 31 +++++++++- .../index_actor/handle_impl.rs | 14 ++++- .../index_controller/index_actor/message.rs | 9 ++- .../src/index_controller/index_actor/mod.rs | 29 ++++----- meilisearch-http/src/index_controller/mod.rs | 57 +++++++++++------- meilisearch-http/src/routes/stats.rs | 59 +++++++++++++++---- 10 files changed, 199 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 461a4789b..a79898d99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1875,6 +1875,23 @@ dependencies = [ "urlencoding", "uuid", "vergen", + "walkdir", +] + +[[package]] +name = "meilisearch-tokenizer" +version = "0.1.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.0#833c48b2ee39071f8b4f51abd15122afdb3c8c06" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", ] [[package]] @@ -2840,6 +2857,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -3717,6 +3743,17 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi 0.3.9", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 3e04b876a..9ab386882 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -62,6 +62,7 @@ thiserror = "1.0.24" tokio = { version = "1", features = ["full"] } uuid = "0.8.2" oxidized-json-checker = "0.3.2" +walkdir = "2.3.2" [dependencies.sentry] default-features = false diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 717d728fc..2d0a543d4 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -1,6 +1,3 @@ -pub mod search; -mod updates; - use std::ops::Deref; use std::sync::Arc; @@ -11,6 +8,9 @@ use crate::index_controller::IndexController; use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::option::Opt; +pub mod search; +mod updates; + #[derive(Clone)] pub struct Data { inner: Arc, diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index dfd2ebdc4..9ba03ddd2 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,6 +1,3 @@ -mod search; -mod updates; - use std::collections::{BTreeSet, HashSet}; use std::ops::Deref; use std::sync::Arc; @@ -8,10 +5,14 @@ use std::sync::Arc; use anyhow::{bail, Context}; use milli::obkv_to_json; use serde_json::{Map, Value}; +use walkdir::WalkDir; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use updates::{Facets, Settings, UpdateResult}; +mod search; +mod updates; + pub type Document = Map; #[derive(Clone)] @@ -126,6 +127,15 @@ impl Index { } } + pub fn size(&self) -> anyhow::Result { + Ok(WalkDir::new(self.env.path()) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len())) + } + fn fields_to_display>( &self, txn: &heed::RoTxn, diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index a4228227f..6d0c1e8cd 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -12,12 +12,15 @@ use tokio::sync::mpsc; use tokio::task::spawn_blocking; use uuid::Uuid; -use super::{IndexError, IndexMeta, IndexMsg, IndexSettings, IndexStore, Result, UpdateResult}; use crate::index::{Document, SearchQuery, SearchResult, Settings}; use crate::index_controller::update_handler::UpdateHandler; -use crate::index_controller::{get_arc_ownership_blocking, updates::Processing, UpdateMeta}; +use crate::index_controller::{ + get_arc_ownership_blocking, updates::Processing, IndexStats, UpdateMeta, +}; use crate::option::IndexerOpts; +use super::{IndexError, IndexMeta, IndexMsg, IndexSettings, IndexStore, Result, UpdateResult}; + pub struct IndexActor { read_receiver: Option>, write_receiver: Option>, @@ -146,6 +149,9 @@ impl IndexActor { Snapshot { uuid, path, ret } => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } + GetStats { uuid, ret } => { + let _ = ret.send(self.handle_get_stats(uuid).await); + } } } @@ -328,4 +334,25 @@ impl IndexActor { Ok(()) } + + async fn handle_get_stats(&self, uuid: Uuid) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + + spawn_blocking(move || { + let rtxn = index.read_txn()?; + + Ok(IndexStats { + size: index.size()?, + number_of_documents: index.number_of_documents(&rtxn)?, + is_indexing: false, // TODO check actual is_indexing + fields_distribution: index.fields_distribution(&rtxn)?, + }) + }) + .await + .map_err(|e| IndexError::Error(e.into()))? + } } diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index dba0f9e60..93406c13b 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -3,12 +3,13 @@ use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; +use crate::index::{Document, SearchQuery, SearchResult, Settings}; +use crate::index_controller::{updates::Processing, UpdateMeta}; +use crate::index_controller::{IndexSettings, IndexStats}; + use super::{ IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore, Result, UpdateResult, }; -use crate::index::{Document, SearchQuery, SearchResult, Settings}; -use crate::index_controller::IndexSettings; -use crate::index_controller::{updates::Processing, UpdateMeta}; #[derive(Clone)] pub struct IndexActorHandleImpl { @@ -121,6 +122,13 @@ impl IndexActorHandle for IndexActorHandleImpl { let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + async fn get_index_stats(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::GetStats { uuid, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } impl IndexActorHandleImpl { diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 46d7f6214..6da0f8628 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -3,9 +3,10 @@ use std::path::PathBuf; use tokio::sync::oneshot; use uuid::Uuid; -use super::{IndexMeta, IndexSettings, Result, UpdateResult}; use crate::index::{Document, SearchQuery, SearchResult, Settings}; -use crate::index_controller::{updates::Processing, UpdateMeta}; +use crate::index_controller::{updates::Processing, IndexStats, UpdateMeta}; + +use super::{IndexMeta, IndexSettings, Result, UpdateResult}; pub enum IndexMsg { CreateIndex { @@ -58,4 +59,8 @@ pub enum IndexMsg { path: PathBuf, ret: oneshot::Sender>, }, + GetStats { + uuid: Uuid, + ret: oneshot::Sender>, + }, } diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 2dc856b80..426eb29e4 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -1,30 +1,30 @@ -mod actor; -mod handle_impl; -mod message; -mod store; - use std::path::PathBuf; use chrono::{DateTime, Utc}; +#[cfg(test)] +use mockall::automock; use serde::{Deserialize, Serialize}; use thiserror::Error; use uuid::Uuid; -use super::IndexSettings; +use actor::IndexActor; +pub use handle_impl::IndexActorHandleImpl; +use message::IndexMsg; +use store::{IndexStore, MapIndexStore}; + use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; use crate::index_controller::{ updates::{Failed, Processed, Processing}, - UpdateMeta, + IndexStats, UpdateMeta, }; -use actor::IndexActor; -use message::IndexMsg; -use store::{IndexStore, MapIndexStore}; -pub use handle_impl::IndexActorHandleImpl; +use super::IndexSettings; -#[cfg(test)] -use mockall::automock; +mod actor; +mod handle_impl; +mod message; +mod store; pub type Result = std::result::Result; type UpdateResult = std::result::Result, Failed>; @@ -33,7 +33,7 @@ type UpdateResult = std::result::Result, Failed, - updated_at: DateTime, + pub updated_at: DateTime, primary_key: Option, } @@ -98,4 +98,5 @@ pub trait IndexActorHandle { async fn get_index_meta(&self, uuid: Uuid) -> Result; async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn get_index_stats(&self, uuid: Uuid) -> Result; } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index b26ab8828..e459af10c 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -1,10 +1,3 @@ -mod index_actor; -mod snapshot; -mod update_actor; -mod update_handler; -mod updates; -mod uuid_resolver; - use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -14,33 +7,40 @@ use anyhow::bail; use futures::stream::StreamExt; use log::info; use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use milli::FieldsDistribution; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::time::sleep; use uuid::Uuid; +use index_actor::IndexActorHandle; +use snapshot::load_snapshot; +use snapshot::SnapshotService; +use update_actor::UpdateActorHandle; +pub use updates::{Failed, Processed, Processing}; +use uuid_resolver::UuidError; +use uuid_resolver::UuidResolverHandle; + use crate::index::{Document, SearchQuery, SearchResult}; use crate::index::{Facets, Settings, UpdateResult}; use crate::option::Opt; -use index_actor::IndexActorHandle; -use snapshot::load_snapshot; -use update_actor::UpdateActorHandle; -use uuid_resolver::UuidResolverHandle; - -use snapshot::SnapshotService; -pub use updates::{Failed, Processed, Processing}; -use uuid_resolver::UuidError; +mod index_actor; +mod snapshot; +mod update_actor; +mod update_handler; +mod updates; +mod uuid_resolver; pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { - uid: String, + pub uid: String, name: String, #[serde(flatten)] - meta: index_actor::IndexMeta, + pub meta: index_actor::IndexMeta, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -63,6 +63,14 @@ pub struct IndexSettings { pub primary_key: Option, } +#[derive(Clone, Debug)] +pub struct IndexStats { + pub size: u64, + pub number_of_documents: u64, + pub is_indexing: bool, + pub fields_distribution: FieldsDistribution, +} + pub struct IndexController { uuid_resolver: uuid_resolver::UuidResolverHandleImpl, index_handle: index_actor::IndexActorHandleImpl, @@ -100,10 +108,11 @@ impl IndexController { update_handle.clone(), Duration::from_secs(options.snapshot_interval_sec), options.snapshot_dir.clone(), - options.db_path - .file_name() - .map(|n| n.to_owned().into_string().expect("invalid path")) - .unwrap_or_else(|| String::from("data.ms")), + options + .db_path + .file_name() + .map(|n| n.to_owned().into_string().expect("invalid path")) + .unwrap_or_else(|| String::from("data.ms")), ); tokio::task::spawn(snapshot_service.run()); @@ -341,6 +350,12 @@ impl IndexController { }; Ok(meta) } + + pub async fn get_stats(&self, uid: String) -> anyhow::Result { + let uuid = self.uuid_resolver.get(uid.clone()).await?; + + Ok(self.index_handle.get_index_stats(uuid).await?) + } } pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index 108c67ca9..bab637cd6 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,18 +1,20 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use actix_web::get; use actix_web::web; use actix_web::HttpResponse; use chrono::{DateTime, Utc}; +use milli::FieldsDistribution; use serde::Serialize; use crate::error::ResponseError; use crate::helpers::Authentication; +use crate::index_controller::IndexStats; use crate::routes::IndexParam; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(index_stats) + cfg.service(get_index_stats) .service(get_stats) .service(get_version); } @@ -22,28 +24,61 @@ pub fn services(cfg: &mut web::ServiceConfig) { struct IndexStatsResponse { number_of_documents: u64, is_indexing: bool, - fields_distribution: BTreeMap, + fields_distribution: FieldsDistribution, +} + +impl From for IndexStatsResponse { + fn from(stats: IndexStats) -> Self { + Self { + number_of_documents: stats.number_of_documents, + is_indexing: stats.is_indexing, + fields_distribution: stats.fields_distribution, + } + } } #[get("/indexes/{index_uid}/stats", wrap = "Authentication::Private")] -async fn index_stats( - _data: web::Data, - _path: web::Path, +async fn get_index_stats( + data: web::Data, + path: web::Path, ) -> Result { - todo!() + let response: IndexStatsResponse = data + .index_controller + .get_stats(path.index_uid.clone()) + .await? + .into(); + + Ok(HttpResponse::Ok().json(response)) } #[derive(Serialize)] #[serde(rename_all = "camelCase")] -struct StatsResult { +struct StatsResponse { database_size: u64, last_update: Option>, indexes: HashMap, } #[get("/stats", wrap = "Authentication::Private")] -async fn get_stats(_data: web::Data) -> Result { - todo!() +async fn get_stats(data: web::Data) -> Result { + let mut response = StatsResponse { + database_size: 0, + last_update: None, + indexes: HashMap::new(), + }; + + for index in data.index_controller.list_indexes().await? { + let stats = data.index_controller.get_stats(index.uid.clone()).await?; + + response.database_size += stats.size; + response.last_update = Some(match response.last_update { + Some(last_update) => last_update.max(index.meta.updated_at), + None => index.meta.updated_at, + }); + response.indexes.insert(index.uid, stats.into()); + } + + Ok(HttpResponse::Ok().json(response)) } #[derive(Serialize)] @@ -58,11 +93,11 @@ struct VersionResponse { async fn get_version() -> HttpResponse { let commit_sha = match option_env!("COMMIT_SHA") { Some("") | None => env!("VERGEN_SHA"), - Some(commit_sha) => commit_sha + Some(commit_sha) => commit_sha, }; let commit_date = match option_env!("COMMIT_DATE") { Some("") | None => env!("VERGEN_COMMIT_DATE"), - Some(commit_date) => commit_date + Some(commit_date) => commit_date, }; HttpResponse::Ok().json(VersionResponse { From 09d9a291762ff31f03b563a552866183f138cdba Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 1 Apr 2021 21:54:37 +0300 Subject: [PATCH 247/522] test(http): server & index stats --- meilisearch-http/tests/common/index.rs | 5 +++ meilisearch-http/tests/common/server.rs | 4 ++ meilisearch-http/tests/index/mod.rs | 1 + meilisearch-http/tests/index/stats.rs | 48 ++++++++++++++++++++++++ meilisearch-http/tests/stats/mod.rs | 50 +++++++++++++++++++++++++ 5 files changed, 108 insertions(+) create mode 100644 meilisearch-http/tests/index/stats.rs diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 78ad846de..67ea6c19a 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -161,6 +161,11 @@ impl Index<'_> { let url = format!("/indexes/{}/settings", self.uid); self.service.delete(url).await } + + pub async fn stats(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/stats", self.uid); + self.service.get(url).await + } } pub struct GetDocumentOptions; diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 4655b10a8..e814ab3ed 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -58,6 +58,10 @@ impl Server { pub async fn version(&self) -> (Value, StatusCode) { self.service.get("/version").await } + + pub async fn stats(&self) -> (Value, StatusCode) { + self.service.get("/stats").await + } } pub fn default_settings(dir: impl AsRef) -> Opt { diff --git a/meilisearch-http/tests/index/mod.rs b/meilisearch-http/tests/index/mod.rs index c9804c160..9996df2e7 100644 --- a/meilisearch-http/tests/index/mod.rs +++ b/meilisearch-http/tests/index/mod.rs @@ -1,4 +1,5 @@ mod create_index; mod delete_index; mod get_index; +mod stats; mod update_index; diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs new file mode 100644 index 000000000..d32c06d2b --- /dev/null +++ b/meilisearch-http/tests/index/stats.rs @@ -0,0 +1,48 @@ +use serde_json::json; + +use crate::common::Server; + +#[actix_rt::test] +async fn stats() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(Some("id")).await; + + assert_eq!(code, 200); + + let (response, code) = index.stats().await; + + assert_eq!(code, 200); + assert_eq!(response["numberOfDocuments"], 0); + assert_eq!(response["isIndexing"], false); + assert!(response["fieldsDistribution"] + .as_object() + .unwrap() + .is_empty()); + + let documents = json!([ + { + "id": 1, + "name": "Alexey", + }, + { + "id": 2, + "age": 45, + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + assert_eq!(response["updateId"], 0); + + index.wait_update_id(0).await; + + let (response, code) = index.stats().await; + + assert_eq!(code, 200); + assert_eq!(response["numberOfDocuments"], 2); + assert_eq!(response["isIndexing"], false); + assert_eq!(response["fieldsDistribution"]["id"], 2); + assert_eq!(response["fieldsDistribution"]["name"], 1); + assert_eq!(response["fieldsDistribution"]["age"], 1); +} diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index db04fb1c0..e5027c71f 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -1,3 +1,5 @@ +use serde_json::json; + use crate::common::Server; #[actix_rt::test] @@ -19,3 +21,51 @@ async fn test_healthyness() { assert_eq!(status_code, 200); assert_eq!(response["status"], "available"); } + +#[actix_rt::test] +async fn stats() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(Some("id")).await; + + assert_eq!(code, 200); + + let (response, code) = server.stats().await; + + assert_eq!(code, 200); + assert!(response.get("databaseSize").is_some()); + assert!(response.get("lastUpdate").is_some()); + assert!(response["indexes"].get("test").is_some()); + assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 0); + assert_eq!(response["indexes"]["test"]["isIndexing"], false); + + let last_update = response["lastUpdate"].as_str().unwrap(); + + let documents = json!([ + { + "id": 1, + "name": "Alexey", + }, + { + "id": 2, + "age": 45, + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + assert_eq!(response["updateId"], 0); + + index.wait_update_id(0).await; + + let (response, code) = server.stats().await; + + assert_eq!(code, 200); + assert!(response["databaseSize"].as_u64().unwrap() > 0); + assert!(response["lastUpdate"].as_str().unwrap() > last_update); + assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 2); + assert_eq!(response["indexes"]["test"]["isIndexing"], false); + assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["id"], 2); + assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["name"], 1); + assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["age"], 1); +} From 87412f63ef102974ab9ac7811060345dd2f357f1 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 2 Apr 2021 14:44:35 +0300 Subject: [PATCH 248/522] feat(http): implement is_indexing for stats --- .../src/index_controller/index_actor/actor.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 8 +++- .../index_controller/update_actor/actor.rs | 13 ++++++ .../update_actor/handle_impl.rs | 45 +++++++++++-------- .../index_controller/update_actor/message.rs | 4 ++ .../src/index_controller/update_actor/mod.rs | 1 + .../src/index_controller/update_handler.rs | 4 +- meilisearch-http/tests/index/stats.rs | 5 +++ meilisearch-http/tests/stats/mod.rs | 5 +++ 9 files changed, 65 insertions(+), 22 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 6d0c1e8cd..d56c53fcd 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -348,7 +348,7 @@ impl IndexActor { Ok(IndexStats { size: index.size()?, number_of_documents: index.number_of_documents(&rtxn)?, - is_indexing: false, // TODO check actual is_indexing + is_indexing: false, // We set this field in src/index_controller/mod.rs get_stats fields_distribution: index.fields_distribution(&rtxn)?, }) }) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index e459af10c..967506a55 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -354,7 +354,13 @@ impl IndexController { pub async fn get_stats(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.get(uid.clone()).await?; - Ok(self.index_handle.get_index_stats(uuid).await?) + let stats = self.index_handle.get_index_stats(uuid); + let is_indexing = self.update_handle.is_locked(uuid); + + Ok(IndexStats { + is_indexing: is_indexing.await?, + ..stats.await? + }) } } diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index d87b910d6..c99c0059f 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -72,6 +72,9 @@ where Some(Snapshot { uuid, path, ret }) => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } + Some(IsLocked { uuid, ret }) => { + let _ = ret.send(self.handle_is_locked(uuid).await); + } None => break, } } @@ -223,4 +226,14 @@ where Ok(()) } + + async fn handle_is_locked(&self, uuid: Uuid) -> Result { + let store = self + .store + .get(uuid) + .await? + .ok_or(UpdateError::UnexistingIndex(uuid))?; + + Ok(store.update_lock.is_locked()) + } } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 59f67fbe0..621675955 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -3,11 +3,12 @@ use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; +use crate::index_controller::IndexActorHandle; + use super::{ MapUpdateStoreStore, PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStatus, }; -use crate::index_controller::IndexActorHandle; #[derive(Clone)] pub struct UpdateActorHandleImpl { @@ -36,6 +37,7 @@ where Ok(Self { sender }) } } + #[async_trait::async_trait] impl UpdateActorHandle for UpdateActorHandleImpl where @@ -43,29 +45,12 @@ where { type Data = D; - async fn update( - &self, - meta: UpdateMeta, - data: mpsc::Receiver>, - uuid: Uuid, - ) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Update { - uuid, - data, - meta, - ret, - }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::ListUpdates { uuid, ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } - async fn update_status(&self, uuid: Uuid, id: u64) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::GetUpdate { uuid, id, ret }; @@ -93,4 +78,28 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + async fn update( + &self, + meta: UpdateMeta, + data: mpsc::Receiver>, + uuid: Uuid, + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Update { + uuid, + data, + meta, + ret, + }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + + async fn is_locked(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::IsLocked { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 8e6e3c212..409fbcebc 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -34,4 +34,8 @@ pub enum UpdateMsg { path: PathBuf, ret: oneshot::Sender>, }, + IsLocked { + uuid: Uuid, + ret: oneshot::Sender>, + }, } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index f3c3caf04..095e068e8 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -52,4 +52,5 @@ pub trait UpdateActorHandle { data: mpsc::Receiver>, uuid: Uuid, ) -> Result; + async fn is_locked(&self, uuid: Uuid) -> Result; } diff --git a/meilisearch-http/src/index_controller/update_handler.rs b/meilisearch-http/src/index_controller/update_handler.rs index 17f7107a2..1eb622cbf 100644 --- a/meilisearch-http/src/index_controller/update_handler.rs +++ b/meilisearch-http/src/index_controller/update_handler.rs @@ -39,7 +39,7 @@ impl UpdateHandler { }) } - fn update_buidler(&self, update_id: u64) -> UpdateBuilder { + fn update_builder(&self, update_id: u64) -> UpdateBuilder { // We prepare the update by using the update builder. let mut update_builder = UpdateBuilder::new(update_id); if let Some(max_nb_chunks) = self.max_nb_chunks { @@ -67,7 +67,7 @@ impl UpdateHandler { let update_id = meta.id(); - let update_builder = self.update_buidler(update_id); + let update_builder = self.update_builder(update_id); let result = match meta.meta() { DocumentsAddition { diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs index d32c06d2b..e1d8bd211 100644 --- a/meilisearch-http/tests/index/stats.rs +++ b/meilisearch-http/tests/index/stats.rs @@ -35,6 +35,11 @@ async fn stats() { assert_eq!(code, 202); assert_eq!(response["updateId"], 0); + let (response, code) = index.stats().await; + + assert_eq!(code, 200); + assert_eq!(response["isIndexing"], true); + index.wait_update_id(0).await; let (response, code) = index.stats().await; diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index e5027c71f..ee10f9708 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -56,6 +56,11 @@ async fn stats() { assert_eq!(code, 202); assert_eq!(response["updateId"], 0); + let (response, code) = server.stats().await; + + assert_eq!(code, 200); + assert_eq!(response["indexes"]["test"]["isIndexing"], true); + index.wait_update_id(0).await; let (response, code) = server.stats().await; From 698a1ea5822c3ca154cf3bfdfb0a8da27660bbfb Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 7 Apr 2021 18:57:46 +0300 Subject: [PATCH 249/522] feat(http): store processing as RwLock> in index_actor --- .../src/index_controller/index_actor/actor.rs | 39 +++++++++++++------ meilisearch-http/src/index_controller/mod.rs | 8 +--- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index d56c53fcd..0620765d5 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -8,7 +8,7 @@ use futures::pin_mut; use futures::stream::StreamExt; use heed::CompactionOption; use log::debug; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, RwLock}; use tokio::task::spawn_blocking; use uuid::Uuid; @@ -25,6 +25,7 @@ pub struct IndexActor { read_receiver: Option>, write_receiver: Option>, update_handler: Arc, + processing: RwLock>, store: S, } @@ -42,8 +43,9 @@ impl IndexActor { Ok(Self { read_receiver, write_receiver, - store, update_handler, + processing: RwLock::new(Default::default()), + store, }) } @@ -181,16 +183,26 @@ impl IndexActor { meta: Processing, data: File, ) -> Result { - debug!("Processing update {}", meta.id()); - let uuid = meta.index_uuid(); - let update_handler = self.update_handler.clone(); - let index = match self.store.get(*uuid).await? { - Some(index) => index, - None => self.store.create(*uuid, None).await?, + let uuid = meta.index_uuid().clone(); + + *self.processing.write().await = Some(uuid); + + let result = { + debug!("Processing update {}", meta.id()); + let update_handler = self.update_handler.clone(); + let index = match self.store.get(uuid).await? { + Some(index) => index, + None => self.store.create(uuid, None).await?, + }; + + spawn_blocking(move || update_handler.handle_update(meta, data, index)) + .await + .map_err(|e| IndexError::Error(e.into())) }; - spawn_blocking(move || update_handler.handle_update(meta, data, index)) - .await - .map_err(|e| IndexError::Error(e.into())) + + *self.processing.write().await = None; + + result } async fn handle_settings(&self, uuid: Uuid) -> Result { @@ -342,13 +354,16 @@ impl IndexActor { .await? .ok_or(IndexError::UnexistingIndex)?; + let processing = self.processing.read().await; + let is_indexing = *processing == Some(uuid); + spawn_blocking(move || { let rtxn = index.read_txn()?; Ok(IndexStats { size: index.size()?, number_of_documents: index.number_of_documents(&rtxn)?, - is_indexing: false, // We set this field in src/index_controller/mod.rs get_stats + is_indexing, fields_distribution: index.fields_distribution(&rtxn)?, }) }) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 967506a55..e459af10c 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -354,13 +354,7 @@ impl IndexController { pub async fn get_stats(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.get(uid.clone()).await?; - let stats = self.index_handle.get_index_stats(uuid); - let is_indexing = self.update_handle.is_locked(uuid); - - Ok(IndexStats { - is_indexing: is_indexing.await?, - ..stats.await? - }) + Ok(self.index_handle.get_index_stats(uuid).await?) } } From ae1655586c907508fd4e6241c40519e12f75e32d Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 8 Apr 2021 16:35:28 +0300 Subject: [PATCH 250/522] fixes after review --- Cargo.lock | 16 -------- meilisearch-http/Cargo.lock | 16 -------- meilisearch-http/src/data/mod.rs | 32 +++++++++++++++- .../src/index_controller/index_actor/actor.rs | 19 +++++----- .../index_controller/update_actor/actor.rs | 13 ------- .../update_actor/handle_impl.rs | 7 ---- .../index_controller/update_actor/message.rs | 4 -- .../src/index_controller/update_actor/mod.rs | 1 - meilisearch-http/src/routes/stats.rs | 37 ++++++++----------- 9 files changed, 56 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a79898d99..7ef514ffe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1894,22 +1894,6 @@ dependencies = [ "whatlang", ] -[[package]] -name = "meilisearch-tokenizer" -version = "0.1.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.0#833c48b2ee39071f8b4f51abd15122afdb3c8c06" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - [[package]] name = "memchr" version = "2.3.4" diff --git a/meilisearch-http/Cargo.lock b/meilisearch-http/Cargo.lock index 0bdc739d5..b9bfd06ac 100644 --- a/meilisearch-http/Cargo.lock +++ b/meilisearch-http/Cargo.lock @@ -1827,22 +1827,6 @@ dependencies = [ "vergen", ] -[[package]] -name = "meilisearch-tokenizer" -version = "0.1.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#147b6154b1b34cb8f5da2df6a416b7da191bc850" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - [[package]] name = "memchr" version = "2.3.4" diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 2d0a543d4..9227454a5 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -4,9 +4,11 @@ use std::sync::Arc; use sha2::Digest; use crate::index::Settings; -use crate::index_controller::IndexController; +use crate::index_controller::{IndexController, IndexStats}; use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::option::Opt; +use std::collections::HashMap; +use chrono::{DateTime, Utc}; pub mod search; mod updates; @@ -37,6 +39,13 @@ pub struct ApiKeys { pub master: Option, } +#[derive(Default)] +pub struct Stats { + pub database_size: u64, + pub last_update: Option>, + pub indexes: HashMap, +} + impl ApiKeys { pub fn generate_missing_api_keys(&mut self) { if let Some(master_key) = &self.master { @@ -104,6 +113,27 @@ impl Data { Ok(meta) } + pub async fn get_index_stats(&self, uid: String) -> anyhow::Result { + Ok(self.index_controller.get_stats(uid).await?) + } + + pub async fn get_stats(&self) -> anyhow::Result { + let mut stats = Stats::default(); + + for index in self.index_controller.list_indexes().await? { + let index_stats = self.index_controller.get_stats(index.uid.clone()).await?; + + stats.database_size += index_stats.size; + stats.last_update = Some(match stats.last_update { + Some(last_update) => last_update.max(index.meta.updated_at), + None => index.meta.updated_at, + }); + stats.indexes.insert(index.uid, index_stats); + } + + Ok(stats) + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 0620765d5..099bb97ca 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -44,7 +44,7 @@ impl IndexActor { read_receiver, write_receiver, update_handler, - processing: RwLock::new(Default::default()), + processing: RwLock::new(None), store, }) } @@ -183,23 +183,22 @@ impl IndexActor { meta: Processing, data: File, ) -> Result { - let uuid = meta.index_uuid().clone(); - - *self.processing.write().await = Some(uuid); - - let result = { + async fn get_result(actor: &IndexActor, meta: Processing, data: File) -> Result { debug!("Processing update {}", meta.id()); - let update_handler = self.update_handler.clone(); - let index = match self.store.get(uuid).await? { + let uuid = *meta.index_uuid(); + let update_handler = actor.update_handler.clone(); + let index = match actor.store.get(uuid).await? { Some(index) => index, - None => self.store.create(uuid, None).await?, + None => actor.store.create(uuid, None).await?, }; spawn_blocking(move || update_handler.handle_update(meta, data, index)) .await .map_err(|e| IndexError::Error(e.into())) - }; + } + *self.processing.write().await = Some(meta.index_uuid().clone()); + let result = get_result(self, meta, data).await; *self.processing.write().await = None; result diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index c99c0059f..d87b910d6 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -72,9 +72,6 @@ where Some(Snapshot { uuid, path, ret }) => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } - Some(IsLocked { uuid, ret }) => { - let _ = ret.send(self.handle_is_locked(uuid).await); - } None => break, } } @@ -226,14 +223,4 @@ where Ok(()) } - - async fn handle_is_locked(&self, uuid: Uuid) -> Result { - let store = self - .store - .get(uuid) - .await? - .ok_or(UpdateError::UnexistingIndex(uuid))?; - - Ok(store.update_lock.is_locked()) - } } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 621675955..8778a3674 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -95,11 +95,4 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } - - async fn is_locked(&self, uuid: Uuid) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::IsLocked { uuid, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } } diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 409fbcebc..8e6e3c212 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -34,8 +34,4 @@ pub enum UpdateMsg { path: PathBuf, ret: oneshot::Sender>, }, - IsLocked { - uuid: Uuid, - ret: oneshot::Sender>, - }, } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 095e068e8..f3c3caf04 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -52,5 +52,4 @@ pub trait UpdateActorHandle { data: mpsc::Receiver>, uuid: Uuid, ) -> Result; - async fn is_locked(&self, uuid: Uuid) -> Result; } diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index bab637cd6..c40a20609 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -12,6 +12,7 @@ use crate::helpers::Authentication; use crate::index_controller::IndexStats; use crate::routes::IndexParam; use crate::Data; +use crate::data::Stats; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(get_index_stats) @@ -42,11 +43,7 @@ async fn get_index_stats( data: web::Data, path: web::Path, ) -> Result { - let response: IndexStatsResponse = data - .index_controller - .get_stats(path.index_uid.clone()) - .await? - .into(); + let response: IndexStatsResponse = data.get_index_stats(path.index_uid.clone()).await?.into(); Ok(HttpResponse::Ok().json(response)) } @@ -59,24 +56,22 @@ struct StatsResponse { indexes: HashMap, } +impl From for StatsResponse { + fn from(stats: Stats) -> Self { + Self { + database_size: stats.database_size, + last_update: stats.last_update, + indexes: stats.indexes + .into_iter() + .map(|(uid, index_stats)| (uid, index_stats.into())) + .collect(), + } + } +} + #[get("/stats", wrap = "Authentication::Private")] async fn get_stats(data: web::Data) -> Result { - let mut response = StatsResponse { - database_size: 0, - last_update: None, - indexes: HashMap::new(), - }; - - for index in data.index_controller.list_indexes().await? { - let stats = data.index_controller.get_stats(index.uid.clone()).await?; - - response.database_size += stats.size; - response.last_update = Some(match response.last_update { - Some(last_update) => last_update.max(index.meta.updated_at), - None => index.meta.updated_at, - }); - response.indexes.insert(index.uid, stats.into()); - } + let response: StatsResponse = data.get_stats().await?.into(); Ok(HttpResponse::Ok().json(response)) } From adfdb99abcdf4a3dea552a5f31d7c72f25b03e1d Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 9 Apr 2021 15:41:24 +0300 Subject: [PATCH 251/522] feat(http): calculate updates' and uuids' dbs size --- meilisearch-http/src/analytics.rs | 10 +++------ meilisearch-http/src/data/mod.rs | 11 ++++++++-- meilisearch-http/src/helpers/env.rs | 16 ++++++++++++++ meilisearch-http/src/helpers/mod.rs | 2 ++ meilisearch-http/src/index/mod.rs | 17 ++++---------- .../src/index_controller/index_actor/actor.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 10 +++++++++ .../index_controller/update_actor/actor.rs | 22 ++++++++++++++++++- .../update_actor/handle_impl.rs | 7 ++++++ .../index_controller/update_actor/message.rs | 4 ++++ .../src/index_controller/update_actor/mod.rs | 1 + .../update_actor/update_store.rs | 17 +++++++++++++- .../index_controller/uuid_resolver/actor.rs | 7 ++++++ .../uuid_resolver/handle_impl.rs | 9 ++++++++ .../index_controller/uuid_resolver/message.rs | 4 ++++ .../src/index_controller/uuid_resolver/mod.rs | 1 + .../index_controller/uuid_resolver/store.rs | 6 +++++ 17 files changed, 121 insertions(+), 25 deletions(-) create mode 100644 meilisearch-http/src/helpers/env.rs diff --git a/meilisearch-http/src/analytics.rs b/meilisearch-http/src/analytics.rs index 379a25030..a01cfabe0 100644 --- a/meilisearch-http/src/analytics.rs +++ b/meilisearch-http/src/analytics.rs @@ -8,6 +8,7 @@ use serde_qs as qs; use siphasher::sip::SipHasher; use walkdir::WalkDir; +use crate::helpers::EnvSizer; use crate::Data; use crate::Opt; @@ -33,12 +34,7 @@ impl EventProperties { } } - let database_size = WalkDir::new(&data.db_path) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len()); + let database_size = data.env.size(); let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp()); @@ -116,7 +112,7 @@ pub fn analytics_sender(data: Data, opt: Opt) { time, app_version, user_properties, - event_properties + event_properties, }; let event = serde_json::to_string(&event).unwrap(); diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 9227454a5..3aee18217 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -1,14 +1,14 @@ +use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; +use chrono::{DateTime, Utc}; use sha2::Digest; use crate::index::Settings; use crate::index_controller::{IndexController, IndexStats}; use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::option::Opt; -use std::collections::HashMap; -use chrono::{DateTime, Utc}; pub mod search; mod updates; @@ -119,15 +119,22 @@ impl Data { pub async fn get_stats(&self) -> anyhow::Result { let mut stats = Stats::default(); + stats.database_size += self.index_controller.get_uuids_size().await?; for index in self.index_controller.list_indexes().await? { let index_stats = self.index_controller.get_stats(index.uid.clone()).await?; stats.database_size += index_stats.size; + stats.database_size += self + .index_controller + .get_updates_size(index.uid.clone()) + .await?; + stats.last_update = Some(match stats.last_update { Some(last_update) => last_update.max(index.meta.updated_at), None => index.meta.updated_at, }); + stats.indexes.insert(index.uid, index_stats); } diff --git a/meilisearch-http/src/helpers/env.rs b/meilisearch-http/src/helpers/env.rs new file mode 100644 index 000000000..9bc81bc69 --- /dev/null +++ b/meilisearch-http/src/helpers/env.rs @@ -0,0 +1,16 @@ +use walkdir::WalkDir; + +pub trait EnvSizer { + fn size(&self) -> u64; +} + +impl EnvSizer for heed::Env { + fn size(&self) -> u64 { + WalkDir::new(self.path()) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len()) + } +} diff --git a/meilisearch-http/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs index d1204908f..a5cddf29c 100644 --- a/meilisearch-http/src/helpers/mod.rs +++ b/meilisearch-http/src/helpers/mod.rs @@ -1,4 +1,6 @@ pub mod authentication; pub mod compression; +mod env; pub use authentication::Authentication; +pub use env::EnvSizer; diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 9ba03ddd2..f8835eceb 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -5,10 +5,10 @@ use std::sync::Arc; use anyhow::{bail, Context}; use milli::obkv_to_json; use serde_json::{Map, Value}; -use walkdir::WalkDir; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use updates::{Facets, Settings, UpdateResult}; +use crate::helpers::EnvSizer; mod search; mod updates; @@ -55,11 +55,7 @@ impl Index { let stop_words = self .stop_words(&txn)? .map(|stop_words| -> anyhow::Result> { - Ok(stop_words - .stream() - .into_strs()? - .into_iter() - .collect()) + Ok(stop_words.stream().into_strs()?.into_iter().collect()) }) .transpose()? .unwrap_or_else(BTreeSet::new); @@ -127,13 +123,8 @@ impl Index { } } - pub fn size(&self) -> anyhow::Result { - Ok(WalkDir::new(self.env.path()) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len())) + pub fn size(&self) -> u64 { + self.env.size() } fn fields_to_display>( diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 099bb97ca..c54b2edd2 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -360,7 +360,7 @@ impl IndexActor { let rtxn = index.read_txn()?; Ok(IndexStats { - size: index.size()?, + size: index.size(), number_of_documents: index.number_of_documents(&rtxn)?, is_indexing, fields_distribution: index.fields_distribution(&rtxn)?, diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index e459af10c..8361c45cc 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -356,6 +356,16 @@ impl IndexController { Ok(self.index_handle.get_index_stats(uuid).await?) } + + pub async fn get_updates_size(&self, uid: String) -> anyhow::Result { + let uuid = self.uuid_resolver.get(uid.clone()).await?; + + Ok(self.update_handle.get_size(uuid).await?) + } + + pub async fn get_uuids_size(&self) -> anyhow::Result { + Ok(self.uuid_resolver.get_size().await?) + } } pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index d87b910d6..f725dda84 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -8,10 +8,11 @@ use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio::sync::mpsc; use uuid::Uuid; -use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStoreStore}; use crate::index_controller::index_actor::IndexActorHandle; use crate::index_controller::{get_arc_ownership_blocking, UpdateMeta, UpdateStatus}; +use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStoreStore}; + pub struct UpdateActor { path: PathBuf, store: S, @@ -72,6 +73,9 @@ where Some(Snapshot { uuid, path, ret }) => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } + Some(GetSize { uuid, ret }) => { + let _ = ret.send(self.handle_get_size(uuid).await); + } None => break, } } @@ -223,4 +227,20 @@ where Ok(()) } + + async fn handle_get_size(&self, uuid: Uuid) -> Result { + let size = match self.store.get(uuid).await? { + Some(update_store) => tokio::task::spawn_blocking(move || -> anyhow::Result { + let txn = update_store.env.read_txn()?; + + update_store.get_size(&txn) + }) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?, + None => 0, + }; + + Ok(size) + } } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 8778a3674..860cc2bc8 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -79,6 +79,13 @@ where receiver.await.expect("update actor killed.") } + async fn get_size(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::GetSize { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 8e6e3c212..f8150c00a 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -34,4 +34,8 @@ pub enum UpdateMsg { path: PathBuf, ret: oneshot::Sender>, }, + GetSize { + uuid: Uuid, + ret: oneshot::Sender>, + }, } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index f3c3caf04..228b47b02 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -46,6 +46,7 @@ pub trait UpdateActorHandle { async fn delete(&self, uuid: Uuid) -> Result<()>; async fn create(&self, uuid: Uuid) -> Result<()>; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn get_size(&self, uuid: Uuid) -> Result; async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index a083eb186..3d6c4e396 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -1,3 +1,4 @@ +use std::fs::File; use std::fs::{copy, create_dir_all, remove_file}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -6,10 +7,10 @@ use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use parking_lot::{Mutex, RwLock}; use serde::{Deserialize, Serialize}; -use std::fs::File; use tokio::sync::mpsc; use uuid::Uuid; +use crate::helpers::EnvSizer; use crate::index_controller::updates::*; type BEU64 = heed::zerocopy::U64; @@ -409,4 +410,18 @@ where Ok(()) } + + pub fn get_size(&self, txn: &heed::RoTxn) -> anyhow::Result { + let mut size = self.env.size(); + + for path in self.pending.iter(txn)? { + let (_, path) = path?; + + if let Ok(metadata) = path.metadata() { + size += metadata.len() + } + } + + Ok(size) + } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index d5cde13e7..27ffaa05e 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -41,6 +41,9 @@ impl UuidResolverActor { Some(SnapshotRequest { path, ret }) => { let _ = ret.send(self.handle_snapshot(path).await); } + Some(GetSize { ret }) => { + let _ = ret.send(self.handle_get_size().await); + } // all senders have been dropped, need to quit. None => break, } @@ -86,6 +89,10 @@ impl UuidResolverActor { self.store.insert(uid, uuid).await?; Ok(()) } + + async fn handle_get_size(&self) -> Result { + self.store.get_size().await + } } fn is_index_uid_valid(uid: &str) -> bool { diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index f8625b379..c522e87e6 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -75,4 +75,13 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .await .expect("Uuid resolver actor has been killed")?) } + + async fn get_size(&self) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::GetSize { ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index 975c709b3..e7d29f05f 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -4,6 +4,7 @@ use tokio::sync::oneshot; use uuid::Uuid; use super::Result; + pub enum UuidResolveMsg { Get { uid: String, @@ -29,4 +30,7 @@ pub enum UuidResolveMsg { path: PathBuf, ret: oneshot::Sender>>, }, + GetSize { + ret: oneshot::Sender>, + }, } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 43cd9995b..33a089ddb 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -30,6 +30,7 @@ pub trait UuidResolverHandle { async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; async fn snapshot(&self, path: PathBuf) -> Result>; + async fn get_size(&self) -> Result; } #[derive(Debug, Error)] diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 435314911..1f057830b 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -8,6 +8,7 @@ use heed::{ use uuid::Uuid; use super::{Result, UuidError, UUID_STORE_SIZE}; +use crate::helpers::EnvSizer; #[async_trait::async_trait] pub trait UuidStore { @@ -19,6 +20,7 @@ pub trait UuidStore { async fn list(&self) -> Result>; async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; async fn snapshot(&self, path: PathBuf) -> Result>; + async fn get_size(&self) -> Result; } pub struct HeedUuidStore { @@ -151,4 +153,8 @@ impl UuidStore for HeedUuidStore { }) .await? } + + async fn get_size(&self) -> Result { + Ok(self.env.size()) + } } From 9eaf048a06e4d18e03b1cfce946c29305c9b9b22 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 13 Apr 2021 11:58:22 +0300 Subject: [PATCH 252/522] fix(http): use BTreeMap instead of HashMap to preserve stats order --- meilisearch-http/src/routes/stats.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index c40a20609..7434f1727 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,18 +1,18 @@ -use std::collections::HashMap; +use std::collections::BTreeMap; +use std::iter::FromIterator; use actix_web::get; use actix_web::web; use actix_web::HttpResponse; use chrono::{DateTime, Utc}; -use milli::FieldsDistribution; use serde::Serialize; +use crate::data::Stats; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::index_controller::IndexStats; use crate::routes::IndexParam; use crate::Data; -use crate::data::Stats; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(get_index_stats) @@ -25,7 +25,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { struct IndexStatsResponse { number_of_documents: u64, is_indexing: bool, - fields_distribution: FieldsDistribution, + fields_distribution: BTreeMap, } impl From for IndexStatsResponse { @@ -33,7 +33,7 @@ impl From for IndexStatsResponse { Self { number_of_documents: stats.number_of_documents, is_indexing: stats.is_indexing, - fields_distribution: stats.fields_distribution, + fields_distribution: BTreeMap::from_iter(stats.fields_distribution.into_iter()), } } } @@ -53,7 +53,7 @@ async fn get_index_stats( struct StatsResponse { database_size: u64, last_update: Option>, - indexes: HashMap, + indexes: BTreeMap, } impl From for StatsResponse { @@ -61,7 +61,8 @@ impl From for StatsResponse { Self { database_size: stats.database_size, last_update: stats.last_update, - indexes: stats.indexes + indexes: stats + .indexes .into_iter() .map(|(uid, index_stats)| (uid, index_stats.into())) .collect(), From f87afbc55853a57a58addf3094d4d8ed91ea3a6e Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 9 Apr 2021 09:03:25 +0300 Subject: [PATCH 253/522] fix(http): commit date & SHA in startup message --- Dockerfile | 3 +-- meilisearch-http/src/main.rs | 16 +++++++++++----- meilisearch-http/src/routes/stats.rs | 4 ++-- meilisearch-http/tests/stats/mod.rs | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index a45d2f7e5..bbaa4a570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,8 +27,7 @@ RUN find . -path "*/src/main.rs" -delete ARG COMMIT_SHA ARG COMMIT_DATE -ENV COMMIT_SHA=${COMMIT_SHA} -ENV COMMIT_DATE=${COMMIT_DATE} +ENV COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE} COPY . . RUN $HOME/.cargo/bin/cargo build --release diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index b04c556db..eab7f4ac6 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -100,6 +100,15 @@ async fn run_http( } pub fn print_launch_resume(opt: &Opt, data: &Data) { + let commit_sha = match option_env!("COMMIT_SHA") { + Some("") | None => env!("VERGEN_SHA"), + Some(commit_sha) => commit_sha + }; + let commit_date = match option_env!("COMMIT_DATE") { + Some("") | None => env!("VERGEN_COMMIT_DATE"), + Some(commit_date) => commit_date + }; + let ascii_name = r#" 888b d888 d8b 888 d8b .d8888b. 888 8888b d8888 Y8P 888 Y8P d88P Y88b 888 @@ -116,11 +125,8 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { eprintln!("Database path:\t\t{:?}", opt.db_path); eprintln!("Server listening on:\t\"http://{}\"", opt.http_addr); eprintln!("Environment:\t\t{:?}", opt.env); - eprintln!("Commit SHA:\t\t{:?}", env!("VERGEN_SHA").to_string()); - eprintln!( - "Build date:\t\t{:?}", - env!("VERGEN_BUILD_TIMESTAMP").to_string() - ); + eprintln!("Commit SHA:\t\t{:?}", commit_sha.to_string()); + eprintln!("Commit date:\t\t{:?}", commit_date.to_string()); eprintln!( "Package version:\t{:?}", env!("CARGO_PKG_VERSION").to_string() diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index 7434f1727..988e6cf40 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -81,7 +81,7 @@ async fn get_stats(data: web::Data) -> Result #[serde(rename_all = "camelCase")] struct VersionResponse { commit_sha: String, - build_date: String, + commit_date: String, pkg_version: String, } @@ -98,7 +98,7 @@ async fn get_version() -> HttpResponse { HttpResponse::Ok().json(VersionResponse { commit_sha: commit_sha.to_string(), - build_date: commit_date.to_string(), + commit_date: commit_date.to_string(), pkg_version: env!("CARGO_PKG_VERSION").to_string(), }) } diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index ee10f9708..ef90dcf7f 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -9,7 +9,7 @@ async fn get_settings_unexisting_index() { assert_eq!(code, 200); let version = response.as_object().unwrap(); assert!(version.get("commitSha").is_some()); - assert!(version.get("buildDate").is_some()); + assert!(version.get("commitDate").is_some()); assert!(version.get("pkgVersion").is_some()); } From b0717b75d94f6f10769edafdb35b81fe8dd7b5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 14 Apr 2021 19:06:18 +0200 Subject: [PATCH 254/522] Update tokenizer to v0.2.1 --- Cargo.lock | 20 ++++++++++++++++++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ef514ffe..e2a98faed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1847,7 +1847,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.1", "memmap", "milli", "mime", @@ -1894,6 +1894,22 @@ dependencies = [ "whatlang", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.2.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "memchr" version = "2.3.4" @@ -1942,7 +1958,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.1.1", "memmap", "num-traits", "obkv", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 9ab386882..efaafd077 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -40,7 +40,7 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.0" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" } memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.0" } mime = "0.3.16" From ec3a08ea0cfd7352ab3dfa1e955059f522a758dd Mon Sep 17 00:00:00 2001 From: tamo Date: Thu, 15 Apr 2021 14:39:33 +0200 Subject: [PATCH 255/522] remove another unused legacy file --- meilisearch-http/src/lib.rs | 1 - meilisearch-http/src/routes/mod.rs | 1 - meilisearch-http/src/routes/stop_words.rs | 46 ----------------------- 3 files changed, 48 deletions(-) delete mode 100644 meilisearch-http/src/routes/stop_words.rs diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 9532273aa..7665b5694 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -35,7 +35,6 @@ macro_rules! create_app { .configure(index::services) .configure(search::services) .configure(settings::services) - .configure(stop_words::services) .configure(synonym::services) .configure(health::services) .configure(stats::services) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 9164f2d2f..f1c559705 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -8,7 +8,6 @@ pub mod key; pub mod search; pub mod settings; pub mod stats; -pub mod stop_words; pub mod synonym; //pub mod dump; diff --git a/meilisearch-http/src/routes/stop_words.rs b/meilisearch-http/src/routes/stop_words.rs deleted file mode 100644 index 8f89b6425..000000000 --- a/meilisearch-http/src/routes/stop_words.rs +++ /dev/null @@ -1,46 +0,0 @@ -use actix_web::{delete, get, post}; -use actix_web::{web, HttpResponse}; -use std::collections::BTreeSet; - -use crate::error::ResponseError; -use crate::helpers::Authentication; -use crate::routes::IndexParam; -use crate::Data; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get).service(update).service(delete); -} - -#[get( - "/indexes/{index_uid}/settings/stop-words", - wrap = "Authentication::Private" -)] -async fn get( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[post( - "/indexes/{index_uid}/settings/stop-words", - wrap = "Authentication::Private" -)] -async fn update( - _data: web::Data, - _path: web::Path, - _body: web::Json>, -) -> Result { - todo!() -} - -#[delete( - "/indexes/{index_uid}/settings/stop-words", - wrap = "Authentication::Private" -)] -async fn delete( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} From 2c380731b9e423d7168de8f7d3c8ae5e080439e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 19 Apr 2021 16:03:39 +0200 Subject: [PATCH 256/522] Update milli version to v0.1.1 --- Cargo.lock | 24 ++++-------------------- meilisearch-http/Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2a98faed..06ff7fc20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1847,7 +1847,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer 0.2.1", + "meilisearch-tokenizer", "memmap", "milli", "mime", @@ -1878,22 +1878,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "meilisearch-tokenizer" -version = "0.1.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.0#833c48b2ee39071f8b4f51abd15122afdb3c8c06" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - [[package]] name = "meilisearch-tokenizer" version = "0.2.1" @@ -1937,8 +1921,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.1.0#2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c" +version = "0.1.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.1.1#f5ec14c54cc3ab76ca581eca9b6e0870f09fd63f" dependencies = [ "anyhow", "bstr", @@ -1958,7 +1942,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer 0.1.1", + "meilisearch-tokenizer", "memmap", "num-traits", "obkv", diff --git a/meilisearch-http/Cargo.lock b/meilisearch-http/Cargo.lock index b9bfd06ac..baca8ef1a 100644 --- a/meilisearch-http/Cargo.lock +++ b/meilisearch-http/Cargo.lock @@ -1854,7 +1854,7 @@ dependencies = [ [[package]] name = "milli" -version = "0.1.0" +version = "0.1.1" source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" dependencies = [ "anyhow", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index efaafd077..c8c4dacb9 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -42,7 +42,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.0" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.1" } mime = "0.3.16" once_cell = "1.5.2" parking_lot = "0.11.1" From 8eceba98d38194baa831353ed13668b5788d0156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Sat, 17 Apr 2021 17:33:36 +0200 Subject: [PATCH 257/522] Handle star in attributes_to_retrieve --- meilisearch-http/src/routes/document.rs | 15 ++++-- .../tests/documents/get_documents.rs | 51 +++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 7b2bb4628..ed5d88230 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -108,9 +108,18 @@ async fn get_all_documents( params: web::Query, ) -> Result { let attributes_to_retrieve = params - .attributes_to_retrieve - .as_ref() - .map(|attrs| attrs.split(',').map(String::from).collect::>()); + .attributes_to_retrieve + .as_ref() + .and_then(|attrs| { + let mut names = Vec::new(); + for name in attrs.split(',').map(String::from) { + if name == "*" { + return None + } + names.push(name); + } + Some(names) + }); match data .retrieve_documents( diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index 73e29576c..24809532e 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -182,6 +182,23 @@ async fn test_get_all_documents_attributes_to_retrieve() { 0 ); + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["wrong"]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 0 + ); + let (response, code) = index .get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec!["name", "tags"]), @@ -198,6 +215,40 @@ async fn test_get_all_documents_attributes_to_retrieve() { .count(), 2 ); + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["*"]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 16 + ); + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["*", "wrong"]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 16 + ); } #[actix_rt::test] From 45665245dcae15e3502e8f58a1cbc5f270754e94 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 19 Apr 2021 20:23:25 +0200 Subject: [PATCH 258/522] bump actix --- Cargo.lock | 4014 ---------------------------------- meilisearch-error/Cargo.toml | 2 +- meilisearch-http/Cargo.toml | 8 +- 3 files changed, 5 insertions(+), 4019 deletions(-) delete mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 06ff7fc20..000000000 --- a/Cargo.lock +++ /dev/null @@ -1,4014 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "actix-codec" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" -dependencies = [ - "bitflags", - "bytes 0.5.6", - "futures-core", - "futures-sink", - "log", - "pin-project 0.4.27", - "tokio 0.2.25", - "tokio-util 0.3.1", -] - -[[package]] -name = "actix-codec" -version = "0.4.0-beta.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e" -dependencies = [ - "bitflags", - "bytes 1.0.1", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.6", - "tokio 1.3.0", - "tokio-util 0.6.4", -] - -[[package]] -name = "actix-connect" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" -dependencies = [ - "actix-codec 0.3.0", - "actix-rt 1.1.1", - "actix-service 1.0.6", - "actix-utils 2.0.0", - "derive_more", - "either", - "futures-util", - "http", - "log", - "trust-dns-proto", - "trust-dns-resolver", -] - -[[package]] -name = "actix-cors" -version = "0.5.4" -source = "git+https://github.com/MarinPostma/actix-extras.git?rev=8f7b1fd#8f7b1fdd7f58f3a15542a27e6b237666bac756d4" -dependencies = [ - "actix-web", - "derive_more", - "futures-util", - "log", - "once_cell", - "tinyvec", -] - -[[package]] -name = "actix-http" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" -dependencies = [ - "actix-codec 0.3.0", - "actix-connect", - "actix-rt 1.1.1", - "actix-service 1.0.6", - "actix-threadpool", - "actix-utils 2.0.0", - "base64 0.13.0", - "bitflags", - "bytes 0.5.6", - "cookie", - "copyless", - "derive_more", - "either", - "encoding_rs", - "futures-channel", - "futures-core", - "futures-util", - "fxhash", - "h2 0.2.7", - "http", - "httparse", - "indexmap", - "itoa", - "language-tags", - "lazy_static", - "log", - "mime", - "percent-encoding", - "pin-project 1.0.5", - "rand 0.7.3", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "sha-1 0.9.4", - "slab", - "time 0.2.25", -] - -[[package]] -name = "actix-http" -version = "3.0.0-beta.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a01f9e0681608afa887d4269a0857ac4226f09ba5ceda25939e8391c9da610a" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "actix-tls", - "actix-utils 3.0.0-beta.2", - "ahash 0.7.2", - "base64 0.13.0", - "bitflags", - "brotli2", - "bytes 1.0.1", - "bytestring", - "cfg-if 1.0.0", - "cookie", - "derive_more", - "encoding_rs", - "flate2", - "futures-core", - "futures-util", - "h2 0.3.1", - "http", - "httparse", - "itoa", - "language-tags", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project 1.0.5", - "rand 0.8.3", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "sha-1 0.9.4", - "smallvec", - "time 0.2.25", - "tokio 1.3.0", -] - -[[package]] -name = "actix-macros" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" -dependencies = [ - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "actix-macros" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" -dependencies = [ - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "actix-router" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" -dependencies = [ - "bytestring", - "http", - "log", - "regex", - "serde", -] - -[[package]] -name = "actix-rt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" -dependencies = [ - "actix-macros 0.1.3", - "actix-threadpool", - "copyless", - "futures-channel", - "futures-util", - "smallvec", - "tokio 0.2.25", -] - -[[package]] -name = "actix-rt" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b4e57bc1a3915e71526d128baf4323700bd1580bc676239e2298a4c5b001f18" -dependencies = [ - "actix-macros 0.2.0", - "futures-core", - "tokio 1.3.0", -] - -[[package]] -name = "actix-server" -version = "2.0.0-beta.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99198727204a48f82559c18e4b0ba3197b97d5f4576a32bdbef371f3b4599c1" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "actix-utils 3.0.0-beta.2", - "futures-core", - "log", - "mio 0.7.9", - "num_cpus", - "slab", - "tokio 1.3.0", -] - -[[package]] -name = "actix-service" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" -dependencies = [ - "futures-util", - "pin-project 0.4.27", -] - -[[package]] -name = "actix-service" -version = "2.0.0-beta.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca9756f4d32984ac454ae3155a276f6be69b424197bd3f0ca3c87cde72f41d63" -dependencies = [ - "futures-core", - "pin-project-lite 0.2.6", -] - -[[package]] -name = "actix-threadpool" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" -dependencies = [ - "derive_more", - "futures-channel", - "lazy_static", - "log", - "num_cpus", - "parking_lot", - "threadpool", -] - -[[package]] -name = "actix-tls" -version = "3.0.0-beta.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b1455e3f7a26d40cfc1080b571f41e8165e5a88e937ed579f7a4b3d55b0370" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "actix-utils 3.0.0-beta.2", - "derive_more", - "futures-core", - "http", - "log", - "tokio-rustls 0.22.0", - "tokio-util 0.6.4", - "webpki-roots 0.21.0", -] - -[[package]] -name = "actix-utils" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" -dependencies = [ - "actix-codec 0.3.0", - "actix-rt 1.1.1", - "actix-service 1.0.6", - "bitflags", - "bytes 0.5.6", - "either", - "futures-channel", - "futures-sink", - "futures-util", - "log", - "pin-project 0.4.27", - "slab", -] - -[[package]] -name = "actix-utils" -version = "3.0.0-beta.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458795e09a29bc5557604f9ff6f32236fd0ee457d631672e4ec8f6a0103bb292" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.6", -] - -[[package]] -name = "actix-web" -version = "4.0.0-beta.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d95e50c9e32e8456220b5804867de76e97a86ab8c38b51c9edcccc0f0fddca7" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-http 3.0.0-beta.4", - "actix-macros 0.2.0", - "actix-router", - "actix-rt 2.1.0", - "actix-server", - "actix-service 2.0.0-beta.4", - "actix-tls", - "actix-utils 3.0.0-beta.2", - "actix-web-codegen", - "ahash 0.7.2", - "awc", - "bytes 1.0.1", - "derive_more", - "either", - "encoding_rs", - "futures-core", - "futures-util", - "log", - "mime", - "pin-project 1.0.5", - "regex", - "rustls 0.19.0", - "serde", - "serde_json", - "serde_urlencoded", - "smallvec", - "socket2", - "time 0.2.25", - "url", -] - -[[package]] -name = "actix-web-codegen" -version = "0.5.0-beta.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "addr2line" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "ahash" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" - -[[package]] -name = "ahash" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" -dependencies = [ - "getrandom 0.2.2", - "once_cell", - "version_check", -] - -[[package]] -name = "aho-corasick" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" -dependencies = [ - "memchr", -] - -[[package]] -name = "ansi_term" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "anyhow" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" - -[[package]] -name = "assert-json-diff" -version = "1.0.1" -source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "async-compression" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72c1f1154e234325b50864a349b9c8e56939e266a4c307c0f159812df2f9537" -dependencies = [ - "flate2", - "futures-core", - "memchr", - "pin-project-lite 0.2.6", - "tokio 0.2.25", -] - -[[package]] -name = "async-stream" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" -dependencies = [ - "async-stream-impl", - "futures-core", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "async-trait" -version = "0.1.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi 0.3.9", -] - -[[package]] -name = "autocfg" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" - -[[package]] -name = "awc" -version = "3.0.0-beta.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09aecd8728f6491a62b27454ea4b36fb7e50faf32928b0369b644e402c651f4e" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-http 3.0.0-beta.4", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "base64 0.13.0", - "bytes 1.0.1", - "cfg-if 1.0.0", - "derive_more", - "futures-core", - "itoa", - "log", - "mime", - "percent-encoding", - "pin-project-lite 0.2.6", - "rand 0.8.3", - "rustls 0.19.0", - "serde", - "serde_json", - "serde_urlencoded", -] - -[[package]] -name = "backtrace" -version = "0.3.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" -dependencies = [ - "addr2line", - "cfg-if 1.0.0", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base-x" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" - -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - -[[package]] -name = "base64" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" - -[[package]] -name = "bincode" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" -dependencies = [ - "byteorder", - "serde", -] - -[[package]] -name = "bitflags" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" - -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding", - "byte-tools", - "byteorder", - "generic-array 0.12.4", -] - -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array 0.14.4", -] - -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", -] - -[[package]] -name = "brotli-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "brotli2" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" -dependencies = [ - "brotli-sys", - "libc", -] - -[[package]] -name = "bstr" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", -] - -[[package]] -name = "bumpalo" -version = "3.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" - -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - -[[package]] -name = "byte-unit" -version = "4.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9520900471c3a9bbcfe0fd4c7b6bcfeff41b20a76cf91c59b7474b09be1ee27" -dependencies = [ - "utf8-width", -] - -[[package]] -name = "bytemuck" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - -[[package]] -name = "bytes" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" - -[[package]] -name = "bytes" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" - -[[package]] -name = "bytestring" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" -dependencies = [ - "bytes 1.0.1", -] - -[[package]] -name = "cc" -version = "1.0.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cedarwood" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963e82c7b94163808ca3a452608d260b64ba5bc7b5653b4af1af59887899f48d" -dependencies = [ - "smallvec", -] - -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "character_converter" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e48477ece09d6a21c033cb604968524a37782532727055d6f6faafac1781e5c" -dependencies = [ - "bincode", -] - -[[package]] -name = "chrono" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" -dependencies = [ - "libc", - "num-integer", - "num-traits", - "serde", - "time 0.1.44", - "winapi 0.3.9", -] - -[[package]] -name = "clap" -version = "2.33.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" -dependencies = [ - "ansi_term", - "atty", - "bitflags", - "strsim", - "textwrap", - "unicode-width", - "vec_map", -] - -[[package]] -name = "const_fn" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" - -[[package]] -name = "cookie" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" -dependencies = [ - "percent-encoding", - "time 0.2.25", - "version_check", -] - -[[package]] -name = "copyless" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" - -[[package]] -name = "cow-utils" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" - -[[package]] -name = "cpuid-bool" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" - -[[package]] -name = "crc32fast" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" -dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" -dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch", - "crossbeam-utils 0.8.3", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" -dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", - "lazy_static", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-queue" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" -dependencies = [ - "crossbeam-utils 0.6.6", -] - -[[package]] -name = "crossbeam-utils" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" -dependencies = [ - "cfg-if 0.1.10", - "lazy_static", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" -dependencies = [ - "autocfg", - "cfg-if 1.0.0", - "lazy_static", -] - -[[package]] -name = "csv" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" -dependencies = [ - "bstr", - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" -dependencies = [ - "memchr", -] - -[[package]] -name = "dashmap" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" -dependencies = [ - "cfg-if 1.0.0", - "num_cpus", -] - -[[package]] -name = "debugid" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba" -dependencies = [ - "serde", - "uuid", -] - -[[package]] -name = "derive_more" -version = "0.99.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "deunicode" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" - -[[package]] -name = "difference" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" - -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.4", -] - -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array 0.14.4", -] - -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - -[[package]] -name = "downcast" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" - -[[package]] -name = "either" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" - -[[package]] -name = "encoding_rs" -version = "0.8.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "enum-as-inner" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" -dependencies = [ - "heck", - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "env_logger" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" -dependencies = [ - "atty", - "humantime 2.1.0", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "failure" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" -dependencies = [ - "backtrace", - "failure_derive", -] - -[[package]] -name = "failure_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", - "synstructure", -] - -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - -[[package]] -name = "filetime" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "redox_syscall", - "winapi 0.3.9", -] - -[[package]] -name = "flate2" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" -dependencies = [ - "cfg-if 1.0.0", - "crc32fast", - "libc", - "miniz_oxide", -] - -[[package]] -name = "float-cmp" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" -dependencies = [ - "num-traits", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "form_urlencoded" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" -dependencies = [ - "matches", - "percent-encoding", -] - -[[package]] -name = "fragile" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" - -[[package]] -name = "fs_extra" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" - -[[package]] -name = "fst" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f" - -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - -[[package]] -name = "futures" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" - -[[package]] -name = "futures-executor" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" - -[[package]] -name = "futures-macro" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" -dependencies = [ - "proc-macro-hack", - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "futures-sink" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" - -[[package]] -name = "futures-task" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" - -[[package]] -name = "futures-util" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite 0.2.6", - "pin-utils", - "proc-macro-hack", - "proc-macro-nested", - "slab", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - -[[package]] -name = "generic-array" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", -] - -[[package]] -name = "gimli" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" - -[[package]] -name = "glob" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" - -[[package]] -name = "grenad" -version = "0.1.0" -source = "git+https://github.com/Kerollmops/grenad.git?rev=3adcb26#3adcb267dcbc590c7da10eb5f887a254865b3dbe" -dependencies = [ - "byteorder", - "flate2", - "log", - "nix", - "snap", - "tempfile", - "zstd", -] - -[[package]] -name = "h2" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" -dependencies = [ - "bytes 0.5.6", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio 0.2.25", - "tokio-util 0.3.1", - "tracing", - "tracing-futures", -] - -[[package]] -name = "h2" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" -dependencies = [ - "bytes 1.0.1", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio 1.3.0", - "tokio-util 0.6.4", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" -dependencies = [ - "ahash 0.3.8", - "autocfg", -] - -[[package]] -name = "hashbrown" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" - -[[package]] -name = "heck" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "heed" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" -dependencies = [ - "byteorder", - "heed-traits", - "heed-types", - "libc", - "lmdb-rkv-sys", - "once_cell", - "page_size", - "serde", - "synchronoise", - "url", - "zerocopy", -] - -[[package]] -name = "heed-traits" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" - -[[package]] -name = "heed-types" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" -dependencies = [ - "bincode", - "heed-traits", - "serde", - "serde_json", - "zerocopy", -] - -[[package]] -name = "hermit-abi" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" -dependencies = [ - "libc", -] - -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi 0.3.9", -] - -[[package]] -name = "http" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" -dependencies = [ - "bytes 1.0.1", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" -dependencies = [ - "bytes 0.5.6", - "http", -] - -[[package]] -name = "httparse" -version = "1.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" - -[[package]] -name = "httpdate" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" - -[[package]] -name = "human_format" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" - -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "0.13.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" -dependencies = [ - "bytes 0.5.6", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.2.7", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project 1.0.5", - "socket2", - "tokio 0.2.25", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" -dependencies = [ - "bytes 0.5.6", - "futures-util", - "hyper", - "log", - "rustls 0.18.1", - "tokio 0.2.25", - "tokio-rustls 0.14.1", - "webpki", -] - -[[package]] -name = "idna" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "im" -version = "14.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "696059c87b83c5a258817ecd67c3af915e3ed141891fc35a1e79908801cf0ce7" -dependencies = [ - "bitmaps", - "rand_core 0.5.1", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - -[[package]] -name = "indexmap" -version = "1.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" -dependencies = [ - "autocfg", - "hashbrown 0.9.1", - "serde", -] - -[[package]] -name = "instant" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - -[[package]] -name = "ipconfig" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" -dependencies = [ - "socket2", - "widestring", - "winapi 0.3.9", - "winreg 0.6.2", -] - -[[package]] -name = "ipnet" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" - -[[package]] -name = "itertools" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" - -[[package]] -name = "jemalloc-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" -dependencies = [ - "cc", - "fs_extra", - "libc", -] - -[[package]] -name = "jemallocator" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" -dependencies = [ - "jemalloc-sys", - "libc", -] - -[[package]] -name = "jieba-rs" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fbdeee8786790f4a99fa30ff5c5f88aa5183f7583693e3788d17fc8a48f33a" -dependencies = [ - "cedarwood", - "fxhash", - "hashbrown 0.9.1", - "lazy_static", - "phf", - "phf_codegen", - "regex", -] - -[[package]] -name = "jobserver" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - -[[package]] -name = "language-tags" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "levenshtein_automata" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a" -dependencies = [ - "fst", -] - -[[package]] -name = "libc" -version = "0.2.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a" - -[[package]] -name = "linked-hash-map" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" - -[[package]] -name = "lmdb-rkv-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b27470ac25167b3afdfb6af8fcd3bc1be67de50ffbdaf4073378cfded6ae24a5" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "lock_api" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "logging_timer" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40d0c249955c17c2f8f86b5f501b16d2509ebbe775f7b1d1d2b1ba85ade2a793" -dependencies = [ - "log", - "logging_timer_proc_macros", -] - -[[package]] -name = "logging_timer_proc_macros" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "482c2c28e6bcfe7c4274f82f701774d755e6aa873edfd619460fcd0966e0eb07" -dependencies = [ - "log", - "proc-macro2 0.4.30", - "quote 0.6.13", - "syn 0.15.44", -] - -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - -[[package]] -name = "main_error" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb63bb1e282e0b6aba0addb1f0e87cb5181ea68142b2dfd21ba108f8e8088a64" - -[[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - -[[package]] -name = "matches" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" - -[[package]] -name = "meilisearch-error" -version = "0.19.0" -dependencies = [ - "actix-http 2.2.0", -] - -[[package]] -name = "meilisearch-http" -version = "0.21.0-alpha.2" -dependencies = [ - "actix-cors", - "actix-http 3.0.0-beta.4", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "actix-web", - "anyhow", - "assert-json-diff", - "async-compression", - "async-stream", - "async-trait", - "byte-unit", - "bytes 0.6.0", - "chrono", - "crossbeam-channel", - "dashmap", - "either", - "env_logger 0.8.3", - "flate2", - "fst", - "futures", - "futures-util", - "grenad", - "heed", - "http", - "indexmap", - "itertools 0.10.0", - "jemallocator", - "log", - "main_error", - "meilisearch-error", - "meilisearch-tokenizer", - "memmap", - "milli", - "mime", - "mockall", - "once_cell", - "oxidized-json-checker", - "parking_lot", - "rand 0.7.3", - "rayon", - "regex", - "rustls 0.19.0", - "sentry", - "serde", - "serde_json", - "serde_url_params", - "sha2", - "siphasher", - "slice-group-by", - "structopt", - "tar", - "tempdir", - "tempfile", - "thiserror", - "tokio 1.3.0", - "urlencoding", - "uuid", - "vergen", - "walkdir", -] - -[[package]] -name = "meilisearch-tokenizer" -version = "0.2.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - -[[package]] -name = "memchr" -version = "2.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" - -[[package]] -name = "memmap" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -dependencies = [ - "libc", - "winapi 0.3.9", -] - -[[package]] -name = "memoffset" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" -dependencies = [ - "autocfg", -] - -[[package]] -name = "milli" -version = "0.1.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.1.1#f5ec14c54cc3ab76ca581eca9b6e0870f09fd63f" -dependencies = [ - "anyhow", - "bstr", - "byteorder", - "chrono", - "crossbeam-channel", - "csv", - "either", - "flate2", - "fst", - "fxhash", - "grenad", - "heed", - "human_format", - "itertools 0.10.0", - "levenshtein_automata", - "linked-hash-map", - "log", - "logging_timer", - "meilisearch-tokenizer", - "memmap", - "num-traits", - "obkv", - "once_cell", - "ordered-float", - "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", - "pest_derive", - "rayon", - "regex", - "roaring", - "serde", - "serde_json", - "slice-group-by", - "smallstr", - "smallvec", - "tempfile", - "tinytemplate", - "uuid", -] - -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - -[[package]] -name = "mime_guess" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "miniz_oxide" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" -dependencies = [ - "adler", - "autocfg", -] - -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow 0.2.2", - "net2", - "slab", - "winapi 0.2.8", -] - -[[package]] -name = "mio" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" -dependencies = [ - "libc", - "log", - "miow 0.3.6", - "ntapi", - "winapi 0.3.9", -] - -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio 0.6.23", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - -[[package]] -name = "miow" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" -dependencies = [ - "socket2", - "winapi 0.3.9", -] - -[[package]] -name = "mockall" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d614ad23f9bb59119b8b5670a85c7ba92c5e9adf4385c81ea00c51c8be33d5" -dependencies = [ - "cfg-if 1.0.0", - "downcast", - "fragile", - "lazy_static", - "mockall_derive", - "predicates", - "predicates-tree", -] - -[[package]] -name = "mockall_derive" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd4234635bca06fc96c7368d038061e0aae1b00a764dc817e900dc974e3deea" -dependencies = [ - "cfg-if 1.0.0", - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "net2" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - -[[package]] -name = "nix" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" -dependencies = [ - "bitflags", - "cc", - "cfg-if 1.0.0", - "libc", -] - -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - -[[package]] -name = "ntapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" -dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "num-integer" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_cpus" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "object" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" - -[[package]] -name = "obkv" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" - -[[package]] -name = "once_cell" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" - -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - -[[package]] -name = "ordered-float" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" -dependencies = [ - "num-traits", -] - -[[package]] -name = "oxidized-json-checker" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938464aebf563f48ab86d1cfc0e2df952985c0b814d3108f41d1b85e7f5b0dac" - -[[package]] -name = "page_size" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" -dependencies = [ - "libc", - "winapi 0.3.9", -] - -[[package]] -name = "parking_lot" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" -dependencies = [ - "cfg-if 1.0.0", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi 0.3.9", -] - -[[package]] -name = "percent-encoding" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" - -[[package]] -name = "pest" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" -dependencies = [ - "ucd-trie", -] - -[[package]] -name = "pest" -version = "2.1.3" -source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" -dependencies = [ - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" -dependencies = [ - "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" -dependencies = [ - "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "pest_meta", - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "pest_meta" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" -dependencies = [ - "maplit", - "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "sha-1 0.8.2", -] - -[[package]] -name = "phf" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_codegen" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" -dependencies = [ - "phf_generator", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" -dependencies = [ - "phf_shared", - "rand 0.7.3", -] - -[[package]] -name = "phf_shared" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" -dependencies = [ - "pin-project-internal 0.4.27", -] - -[[package]] -name = "pin-project" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" -dependencies = [ - "pin-project-internal 1.0.5", -] - -[[package]] -name = "pin-project-internal" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "pin-project-lite" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" - -[[package]] -name = "pin-project-lite" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" - -[[package]] -name = "ppv-lite86" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" - -[[package]] -name = "predicates" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" -dependencies = [ - "difference", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" - -[[package]] -name = "predicates-tree" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" -dependencies = [ - "predicates-core", - "treeline", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "version_check", -] - -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - -[[package]] -name = "proc-macro-nested" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" - -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid 0.1.0", -] - -[[package]] -name = "proc-macro2" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" -dependencies = [ - "unicode-xid 0.2.1", -] - -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - -[[package]] -name = "quote" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" -dependencies = [ - "proc-macro2 1.0.24", -] - -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi 0.3.9", -] - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc 0.2.0", - "rand_pcg", -] - -[[package]] -name = "rand" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" -dependencies = [ - "libc", - "rand_chacha 0.3.0", - "rand_core 0.6.2", - "rand_hc 0.3.0", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - -[[package]] -name = "rand_chacha" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.2", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - -[[package]] -name = "rand_core" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" -dependencies = [ - "getrandom 0.2.2", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_hc" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" -dependencies = [ - "rand_core 0.6.2", -] - -[[package]] -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_xoshiro" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rayon" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" -dependencies = [ - "autocfg", - "crossbeam-deque", - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils 0.8.3", - "lazy_static", - "num_cpus", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "redox_syscall" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" -dependencies = [ - "byteorder", -] - -[[package]] -name = "regex-syntax" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "reqwest" -version = "0.10.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" -dependencies = [ - "base64 0.13.0", - "bytes 0.5.6", - "encoding_rs", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "hyper-rustls", - "ipnet", - "js-sys", - "lazy_static", - "log", - "mime", - "mime_guess", - "percent-encoding", - "pin-project-lite 0.2.6", - "rustls 0.18.1", - "serde", - "serde_json", - "serde_urlencoded", - "tokio 0.2.25", - "tokio-rustls 0.14.1", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots 0.20.0", - "winreg 0.7.0", -] - -[[package]] -name = "resolv-conf" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" -dependencies = [ - "hostname", - "quick-error", -] - -[[package]] -name = "retain_mut" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi 0.3.9", -] - -[[package]] -name = "roaring" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6744a4a918e91359ad1d356a91e2e943a86d9fb9ae77f715d617032ea2af88f" -dependencies = [ - "bytemuck", - "byteorder", - "retain_mut", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" - -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - -[[package]] -name = "rustc_version" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" -dependencies = [ - "semver 0.11.0", -] - -[[package]] -name = "rustls" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" -dependencies = [ - "base64 0.12.3", - "log", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustls" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" -dependencies = [ - "base64 0.13.0", - "log", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "ryu" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "sct" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser 0.7.0", -] - -[[package]] -name = "semver" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" -dependencies = [ - "semver-parser 0.10.2", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - -[[package]] -name = "semver-parser" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" -dependencies = [ - "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "sentry" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b01b723fc1b0a0f9394ca1a8451daec6e20206d47f96c3dceea7fd11ec9eec0" -dependencies = [ - "backtrace", - "env_logger 0.7.1", - "failure", - "hostname", - "httpdate", - "im", - "lazy_static", - "libc", - "log", - "rand 0.7.3", - "regex", - "reqwest", - "rustc_version 0.2.3", - "sentry-types", - "uname", - "url", -] - -[[package]] -name = "sentry-types" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ec406c11c060c8a7d5d67fc6f4beb2888338dcb12b9af409451995f124749d" -dependencies = [ - "chrono", - "debugid", - "failure", - "serde", - "serde_json", - "url", - "uuid", -] - -[[package]] -name = "serde" -version = "1.0.124" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.124" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "serde_json" -version = "1.0.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_url_params" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c43307d0640738af32fe8d01e47119bc0fc8a686be470a44a586caff76dfb34" -dependencies = [ - "serde", - "url", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha-1" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", -] - -[[package]] -name = "sha-1" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpuid-bool", - "digest 0.9.0", - "opaque-debug 0.3.0", -] - -[[package]] -name = "sha1" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" - -[[package]] -name = "sha2" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpuid-bool", - "digest 0.9.0", - "opaque-debug 0.3.0", -] - -[[package]] -name = "signal-hook-registry" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" -dependencies = [ - "libc", -] - -[[package]] -name = "siphasher" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" - -[[package]] -name = "sized-chunks" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59044ea371ad781ff976f7b06480b9f0180e834eda94114f2afb4afc12b7718" -dependencies = [ - "bitmaps", - "typenum", -] - -[[package]] -name = "slab" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" - -[[package]] -name = "slice-group-by" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7474f0b646d228360ab62ed974744617bc869d959eac8403bfa3665931a7fb" - -[[package]] -name = "smallstr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e922794d168678729ffc7e07182721a14219c65814e66e91b839a272fe5ae4f" -dependencies = [ - "serde", - "smallvec", -] - -[[package]] -name = "smallvec" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" - -[[package]] -name = "snap" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a" - -[[package]] -name = "socket2" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "winapi 0.3.9", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "standback" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2beb4d1860a61f571530b3f855a1b538d0200f7871c63331ecd6f17b1f014f8" -dependencies = [ - "version_check", -] - -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "serde", - "serde_derive", - "syn 1.0.64", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2 1.0.24", - "quote 1.0.9", - "serde", - "serde_derive", - "serde_json", - "sha1", - "syn 1.0.64", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" - -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "structopt" -version = "0.3.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" -dependencies = [ - "clap", - "lazy_static", - "structopt-derive", -] - -[[package]] -name = "structopt-derive" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid 0.1.0", -] - -[[package]] -name = "syn" -version = "1.0.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "unicode-xid 0.2.1", -] - -[[package]] -name = "synchronoise" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb" -dependencies = [ - "crossbeam-queue", -] - -[[package]] -name = "synstructure" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", - "unicode-xid 0.2.1", -] - -[[package]] -name = "tar" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0bcfbd6a598361fda270d82469fff3d65089dc33e175c9a131f7b4cd395f228" -dependencies = [ - "filetime", - "libc", - "xattr", -] - -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] - -[[package]] -name = "tempfile" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "rand 0.8.3", - "redox_syscall", - "remove_dir_all", - "winapi 0.3.9", -] - -[[package]] -name = "termcolor" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "thiserror" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - -[[package]] -name = "time" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi 0.3.9", -] - -[[package]] -name = "time" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1195b046942c221454c2539395f85413b33383a067449d78aab2b7b052a142f7" -dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", - "time-macros", - "version_check", - "winapi 0.3.9", -] - -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - -[[package]] -name = "time-macros-impl" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" -dependencies = [ - "proc-macro-hack", - "proc-macro2 1.0.24", - "quote 1.0.9", - "standback", - "syn 1.0.64", -] - -[[package]] -name = "tinytemplate" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "tinyvec" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" - -[[package]] -name = "tokio" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" -dependencies = [ - "bytes 0.5.6", - "fnv", - "futures-core", - "iovec", - "lazy_static", - "libc", - "memchr", - "mio 0.6.23", - "mio-uds", - "num_cpus", - "pin-project-lite 0.1.12", - "signal-hook-registry", - "slab", - "winapi 0.3.9", -] - -[[package]] -name = "tokio" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda" -dependencies = [ - "autocfg", - "bytes 1.0.1", - "libc", - "memchr", - "mio 0.7.9", - "num_cpus", - "once_cell", - "parking_lot", - "pin-project-lite 0.2.6", - "signal-hook-registry", - "tokio-macros", - "winapi 0.3.9", -] - -[[package]] -name = "tokio-macros" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", -] - -[[package]] -name = "tokio-rustls" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" -dependencies = [ - "futures-core", - "rustls 0.18.1", - "tokio 0.2.25", - "webpki", -] - -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls 0.19.0", - "tokio 1.3.0", - "webpki", -] - -[[package]] -name = "tokio-util" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" -dependencies = [ - "bytes 0.5.6", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.1.12", - "tokio 0.2.25", -] - -[[package]] -name = "tokio-util" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec31e5cc6b46e653cf57762f36f71d5e6386391d88a72fd6db4508f8f676fb29" -dependencies = [ - "bytes 1.0.1", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.6", - "tokio 1.3.0", -] - -[[package]] -name = "tower-service" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" - -[[package]] -name = "tracing" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" -dependencies = [ - "cfg-if 1.0.0", - "log", - "pin-project-lite 0.2.6", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project 1.0.5", - "tracing", -] - -[[package]] -name = "treeline" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" - -[[package]] -name = "trust-dns-proto" -version = "0.19.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cad71a0c0d68ab9941d2fb6e82f8fb2e86d9945b94e1661dd0aaea2b88215a9" -dependencies = [ - "async-trait", - "cfg-if 1.0.0", - "enum-as-inner", - "futures", - "idna", - "lazy_static", - "log", - "rand 0.7.3", - "smallvec", - "thiserror", - "tokio 0.2.25", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.19.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710f593b371175db53a26d0b38ed2978fafb9e9e8d3868b1acd753ea18df0ceb" -dependencies = [ - "cfg-if 0.1.10", - "futures", - "ipconfig", - "lazy_static", - "log", - "lru-cache", - "resolv-conf", - "smallvec", - "thiserror", - "tokio 0.2.25", - "trust-dns-proto", -] - -[[package]] -name = "try-lock" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" - -[[package]] -name = "typenum" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" - -[[package]] -name = "ucd-trie" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" - -[[package]] -name = "uname" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" -dependencies = [ - "libc", -] - -[[package]] -name = "unicase" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" -dependencies = [ - "matches", -] - -[[package]] -name = "unicode-normalization" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" - -[[package]] -name = "unicode-width" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" - -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - -[[package]] -name = "unicode-xid" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - -[[package]] -name = "url" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" -dependencies = [ - "form_urlencoded", - "idna", - "matches", - "percent-encoding", - "serde", -] - -[[package]] -name = "urlencoding" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" - -[[package]] -name = "utf8-width" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" - -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom 0.2.2", - "serde", -] - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - -[[package]] -name = "vergen" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" -dependencies = [ - "bitflags", - "chrono", - "rustc_version 0.3.3", -] - -[[package]] -name = "version_check" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" - -[[package]] -name = "walkdir" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" -dependencies = [ - "same-file", - "winapi 0.3.9", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log", - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - -[[package]] -name = "wasm-bindgen" -version = "0.2.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" -dependencies = [ - "cfg-if 1.0.0", - "serde", - "serde_json", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" -dependencies = [ - "bumpalo", - "lazy_static", - "log", - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e67a5806118af01f0d9045915676b22aaebecf4178ae7021bc171dab0b897ab" -dependencies = [ - "cfg-if 1.0.0", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" -dependencies = [ - "quote 1.0.9", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.64", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" - -[[package]] -name = "web-sys" -version = "0.3.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki-roots" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" -dependencies = [ - "webpki", -] - -[[package]] -name = "webpki-roots" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" -dependencies = [ - "webpki", -] - -[[package]] -name = "whatlang" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075" -dependencies = [ - "hashbrown 0.7.2", -] - -[[package]] -name = "widestring" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" - -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "winreg" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" -dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "winreg" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" -dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - -[[package]] -name = "xattr" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" -dependencies = [ - "libc", -] - -[[package]] -name = "zerocopy" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" -dependencies = [ - "proc-macro2 1.0.24", - "syn 1.0.64", - "synstructure", -] - -[[package]] -name = "zstd" -version = "0.5.4+zstd.1.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "2.0.6+zstd.1.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "1.4.18+zstd.1.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" -dependencies = [ - "cc", - "glob", - "itertools 0.9.0", - "libc", -] diff --git a/meilisearch-error/Cargo.toml b/meilisearch-error/Cargo.toml index d5a474ea5..7673e1391 100644 --- a/meilisearch-error/Cargo.toml +++ b/meilisearch-error/Cargo.toml @@ -5,4 +5,4 @@ authors = ["marin "] edition = "2018" [dependencies] -actix-http = "2.2.0" +actix-http = "=3.0.0-beta.5" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index c8c4dacb9..e1d46c8e2 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -13,10 +13,10 @@ path = "src/main.rs" vergen = "3.1.0" [dependencies] -actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "8f7b1fd" } -actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } -actix-service = "2.0.0-beta.4" -actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } +actix-cors = "0.6.0-beta.1" +actix-http = { version = "=3.0.0-beta.5" } +actix-service = "=2.0.0-beta.5" +actix-web = { version = "=4.0.0-beta.5", features = ["rustls"] } anyhow = "1.0.36" async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } async-stream = "0.3.0" From ec230c28357acd2c3001b430d7754c7158b67d08 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 29 Mar 2021 09:22:36 +0200 Subject: [PATCH 259/522] enable distinct --- meilisearch-http/Cargo.lock => Cargo.lock | 1414 ++++++++++----------- meilisearch-http/src/index/mod.rs | 4 + meilisearch-http/src/index/updates.rs | 20 +- 3 files changed, 693 insertions(+), 745 deletions(-) rename meilisearch-http/Cargo.lock => Cargo.lock (79%) diff --git a/meilisearch-http/Cargo.lock b/Cargo.lock similarity index 79% rename from meilisearch-http/Cargo.lock rename to Cargo.lock index baca8ef1a..a5baa6d8d 100644 --- a/meilisearch-http/Cargo.lock +++ b/Cargo.lock @@ -1,23 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - -[[package]] -name = "actix-codec" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" -dependencies = [ - "bitflags", - "bytes 0.5.6", - "futures-core", - "futures-sink", - "log", - "pin-project 0.4.27", - "tokio 0.2.24", - "tokio-util 0.3.1", -] - [[package]] name = "actix-codec" version = "0.4.0-beta.1" @@ -29,34 +11,18 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.0", - "tokio 1.2.0", - "tokio-util 0.6.3", -] - -[[package]] -name = "actix-connect" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" -dependencies = [ - "actix-codec 0.3.0", - "actix-rt 1.1.1", - "actix-service 1.0.6", - "actix-utils 2.0.0", - "derive_more", - "either", - "futures-util", - "http", - "log", - "trust-dns-proto", - "trust-dns-resolver", + "pin-project-lite 0.2.6", + "tokio 1.5.0", + "tokio-util 0.6.6", ] [[package]] name = "actix-cors" -version = "0.5.4" +version = "0.6.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa50c395d15e5946cf04bccb1edef583e9fd42aa6710a1f89a725c5e2c4c5503" dependencies = [ + "actix-service", "actix-web", "derive_more", "futures-util", @@ -67,102 +33,48 @@ dependencies = [ [[package]] name = "actix-http" -version = "2.2.0" +version = "3.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" +checksum = "fb9c5d7ceb490d6565156ae1d4d467db17da1759425c65a2e36ac5e182e014e2" dependencies = [ - "actix-codec 0.3.0", - "actix-connect", - "actix-rt 1.1.1", - "actix-service 1.0.6", - "actix-threadpool", - "actix-utils 2.0.0", - "base64 0.13.0", - "bitflags", - "bytes 0.5.6", - "cookie", - "copyless", - "derive_more", - "either", - "encoding_rs", - "futures-channel", - "futures-core", - "futures-util", - "fxhash", - "h2 0.2.7", - "http", - "httparse", - "indexmap", - "itoa", - "language-tags", - "lazy_static", - "log", - "mime", - "percent-encoding", - "pin-project 1.0.2", - "rand 0.7.3", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "sha-1 0.9.2", - "slab", - "time 0.2.23", -] - -[[package]] -name = "actix-http" -version = "3.0.0-beta.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a01f9e0681608afa887d4269a0857ac4226f09ba5ceda25939e8391c9da610a" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", + "actix-codec", + "actix-rt", + "actix-service", "actix-tls", - "actix-utils 3.0.0-beta.2", - "ahash 0.7.0", + "actix-utils", + "ahash 0.7.2", "base64 0.13.0", "bitflags", "brotli2", "bytes 1.0.1", "bytestring", - "cfg-if 1.0.0", "cookie", "derive_more", "encoding_rs", "flate2", "futures-core", "futures-util", - "h2 0.3.1", + "h2 0.3.2", "http", "httparse", "itoa", "language-tags", + "local-channel", "log", "mime", "once_cell", "percent-encoding", - "pin-project 1.0.2", + "pin-project", + "pin-project-lite 0.2.6", "rand 0.8.3", "regex", "serde", "serde_json", "serde_urlencoded", - "sha-1 0.9.2", + "sha-1 0.9.4", "smallvec", - "time 0.2.23", - "tokio 1.2.0", -] - -[[package]] -name = "actix-macros" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" -dependencies = [ - "quote", - "syn", + "time 0.2.26", + "tokio 1.5.0", ] [[package]] @@ -171,8 +83,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ - "quote", - "syn", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] @@ -190,172 +102,106 @@ dependencies = [ [[package]] name = "actix-rt" -version = "1.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" +checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" dependencies = [ - "actix-macros 0.1.3", - "actix-threadpool", - "copyless", - "futures-channel", - "futures-util", - "smallvec", - "tokio 0.2.24", -] - -[[package]] -name = "actix-rt" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b4e57bc1a3915e71526d128baf4323700bd1580bc676239e2298a4c5b001f18" -dependencies = [ - "actix-macros 0.2.0", + "actix-macros", "futures-core", - "tokio 1.2.0", + "tokio 1.5.0", ] [[package]] name = "actix-server" -version = "2.0.0-beta.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99198727204a48f82559c18e4b0ba3197b97d5f4576a32bdbef371f3b4599c1" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "actix-utils 3.0.0-beta.2", - "futures-core", - "log", - "mio 0.7.9", - "num_cpus", - "slab", - "tokio 1.2.0", -] - -[[package]] -name = "actix-service" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" -dependencies = [ - "futures-util", - "pin-project 0.4.27", -] - -[[package]] -name = "actix-service" version = "2.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca9756f4d32984ac454ae3155a276f6be69b424197bd3f0ca3c87cde72f41d63" +checksum = "0872f02a1871257ef09c5a269dce5dc5fea5ccf502adbf5d39f118913b61411c" dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", "futures-core", - "pin-project-lite 0.2.0", + "log", + "mio 0.7.11", + "num_cpus", + "slab", + "tokio 1.5.0", ] [[package]] -name = "actix-threadpool" -version = "0.3.3" +name = "actix-service" +version = "2.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" +checksum = "cf82340ad9f4e4caf43737fd3bbc999778a268015cdc54675f60af6240bd2b05" dependencies = [ - "derive_more", - "futures-channel", - "lazy_static", - "log", - "num_cpus", - "parking_lot", - "threadpool", + "futures-core", + "pin-project-lite 0.2.6", ] [[package]] name = "actix-tls" -version = "3.0.0-beta.4" +version = "3.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b1455e3f7a26d40cfc1080b571f41e8165e5a88e937ed579f7a4b3d55b0370" +checksum = "65b7bb60840962ef0332f7ea01a57d73a24d2cb663708511ff800250bbfef569" dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "actix-utils 3.0.0-beta.2", + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", "derive_more", "futures-core", "http", "log", "tokio-rustls 0.22.0", - "tokio-util 0.6.3", - "webpki-roots 0.21.0", + "tokio-util 0.6.6", + "webpki-roots 0.21.1", ] [[package]] name = "actix-utils" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" +checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" dependencies = [ - "actix-codec 0.3.0", - "actix-rt 1.1.1", - "actix-service 1.0.6", - "bitflags", - "bytes 0.5.6", - "either", - "futures-channel", - "futures-sink", - "futures-util", - "log", - "pin-project 0.4.27", - "slab", -] - -[[package]] -name = "actix-utils" -version = "3.0.0-beta.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458795e09a29bc5557604f9ff6f32236fd0ee457d631672e4ec8f6a0103bb292" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.0", + "local-waker", + "pin-project-lite 0.2.6", ] [[package]] name = "actix-web" -version = "4.0.0-beta.4" +version = "4.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d95e50c9e32e8456220b5804867de76e97a86ab8c38b51c9edcccc0f0fddca7" +checksum = "6de19cc341c2e68b1ee126de171e86b610b5bbcecc660d1250ebed73e0257bd6" dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-http 3.0.0-beta.4", - "actix-macros 0.2.0", + "actix-codec", + "actix-http", + "actix-macros", "actix-router", - "actix-rt 2.1.0", + "actix-rt", "actix-server", - "actix-service 2.0.0-beta.4", + "actix-service", "actix-tls", - "actix-utils 3.0.0-beta.2", + "actix-utils", "actix-web-codegen", - "ahash 0.7.0", - "awc", + "ahash 0.7.2", "bytes 1.0.1", "derive_more", "either", "encoding_rs", "futures-core", "futures-util", + "language-tags", "log", "mime", - "pin-project 1.0.2", + "once_cell", + "pin-project", "regex", - "rustls 0.19.0", "serde", "serde_json", "serde_urlencoded", "smallvec", - "socket2", - "time 0.2.23", + "socket2 0.4.0", + "time 0.2.26", "url", ] @@ -365,25 +211,25 @@ version = "0.5.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] name = "addr2line" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" dependencies = [ "gimli", ] [[package]] name = "adler" -version = "0.2.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" @@ -393,9 +239,9 @@ checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" [[package]] name = "ahash" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa60d2eadd8b12a996add391db32bd1153eac697ba4869660c0016353611426" +checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" dependencies = [ "getrandom 0.2.2", "once_cell", @@ -422,9 +268,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" [[package]] name = "assert-json-diff" @@ -437,15 +283,15 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a" +checksum = "b72c1f1154e234325b50864a349b9c8e56939e266a4c307c0f159812df2f9537" dependencies = [ "flate2", "futures-core", "memchr", - "pin-project-lite 0.1.11", - "tokio 0.2.24", + "pin-project-lite 0.2.6", + "tokio 0.2.25", ] [[package]] @@ -464,20 +310,20 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] name = "async-trait" -version = "0.1.42" +version = "0.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] @@ -497,38 +343,11 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" -[[package]] -name = "awc" -version = "3.0.0-beta.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09aecd8728f6491a62b27454ea4b36fb7e50faf32928b0369b644e402c651f4e" -dependencies = [ - "actix-codec 0.4.0-beta.1", - "actix-http 3.0.0-beta.4", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", - "base64 0.13.0", - "bytes 1.0.1", - "cfg-if 1.0.0", - "derive_more", - "futures-core", - "itoa", - "log", - "mime", - "percent-encoding", - "pin-project-lite 0.2.0", - "rand 0.8.3", - "rustls 0.19.0", - "serde", - "serde_json", - "serde_urlencoded", -] - [[package]] name = "backtrace" -version = "0.3.55" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -558,11 +377,10 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bincode" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "byteorder", "serde", ] @@ -590,7 +408,7 @@ dependencies = [ "block-padding", "byte-tools", "byteorder", - "generic-array 0.12.3", + "generic-array 0.12.4", ] [[package]] @@ -645,9 +463,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.4.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" [[package]] name = "byte-tools" @@ -657,9 +475,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byte-unit" -version = "4.0.9" +version = "4.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8758c32833faaae35b24a73d332e62d0528e89076ae841c63940e37008b153" +checksum = "b9520900471c3a9bbcfe0fd4c7b6bcfeff41b20a76cf91c59b7474b09be1ee27" dependencies = [ "utf8-width", ] @@ -672,9 +490,9 @@ checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" [[package]] name = "byteorder" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" @@ -705,9 +523,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" dependencies = [ "jobserver", ] @@ -773,27 +591,27 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" +checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "cookie" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ "percent-encoding", - "time 0.2.23", + "time 0.2.26", "version_check", ] -[[package]] -name = "copyless" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" - [[package]] name = "cow-utils" version = "0.1.2" @@ -817,12 +635,12 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", ] [[package]] @@ -833,18 +651,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", ] [[package]] name = "crossbeam-epoch" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" dependencies = [ "cfg-if 1.0.0", - "const_fn", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", "lazy_static", "memoffset", "scopeguard", @@ -871,9 +688,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -882,9 +699,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" dependencies = [ "bstr", "csv-core", @@ -924,20 +741,27 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.11" +version = "0.99.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +checksum = "f82b1b72f1263f214c0f823371768776c4f5841b942c9883aa8e5ec584fd0ba6" dependencies = [ - "proc-macro2", - "quote", - "syn", + "convert_case", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] name = "deunicode" -version = "1.1.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" +checksum = "2044dd33b8b682eedf68271c913235cbefdaf17bf842310be1389c21d6e5945c" + +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" [[package]] name = "digest" @@ -945,7 +769,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" dependencies = [ - "generic-array 0.12.3", + "generic-array 0.12.4", ] [[package]] @@ -963,6 +787,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "downcast" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" + [[package]] name = "either" version = "1.6.1" @@ -971,25 +801,13 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.26" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "enum-as-inner" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "env_logger" version = "0.7.1" @@ -1005,12 +823,12 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" dependencies = [ "atty", - "humantime 2.0.1", + "humantime 2.1.0", "log", "regex", "termcolor", @@ -1032,9 +850,9 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", "synstructure", ] @@ -1046,13 +864,13 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "filetime" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "winapi 0.3.9", ] @@ -1068,6 +886,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1076,14 +903,20 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", "percent-encoding", ] +[[package]] +name = "fragile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" + [[package]] name = "fs_extra" version = "1.2.0" @@ -1120,9 +953,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" dependencies = [ "futures-channel", "futures-core", @@ -1135,9 +968,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" dependencies = [ "futures-core", "futures-sink", @@ -1145,15 +978,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" +checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" [[package]] name = "futures-executor" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" dependencies = [ "futures-core", "futures-task", @@ -1162,42 +995,39 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" +checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" [[package]] name = "futures-macro" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] name = "futures-sink" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" +checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" [[package]] name = "futures-task" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" -dependencies = [ - "once_cell", -] +checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" [[package]] name = "futures-util" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" dependencies = [ "futures-channel", "futures-core", @@ -1206,7 +1036,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.2", + "pin-project-lite 0.2.6", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -1224,9 +1054,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" dependencies = [ "typenum", ] @@ -1243,11 +1073,11 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1303,7 +1133,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-util 0.3.1", "tracing", "tracing-futures", @@ -1311,9 +1141,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" +checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00" dependencies = [ "bytes 1.0.1", "fnv", @@ -1323,8 +1153,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.2.0", - "tokio-util 0.6.3", + "tokio 1.5.0", + "tokio-util 0.6.6", "tracing", ] @@ -1344,6 +1174,12 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + [[package]] name = "heck" version = "0.3.2" @@ -1393,9 +1229,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] @@ -1413,9 +1249,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" dependencies = [ "bytes 1.0.1", "fnv", @@ -1434,9 +1270,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" [[package]] name = "httpdate" @@ -1461,15 +1297,15 @@ dependencies = [ [[package]] name = "humantime" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.13.9" +version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" dependencies = [ "bytes 0.5.6", "futures-channel", @@ -1481,9 +1317,9 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.2", - "socket2", - "tokio 0.2.24", + "pin-project", + "socket2 0.3.19", + "tokio 0.2.25", "tower-service", "tracing", "want", @@ -1500,16 +1336,16 @@ dependencies = [ "hyper", "log", "rustls 0.18.1", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-rustls 0.14.1", "webpki", ] [[package]] name = "idna" -version = "0.2.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -1532,9 +1368,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", "hashbrown 0.9.1", @@ -1559,18 +1395,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ipconfig" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" -dependencies = [ - "socket2", - "widestring", - "winapi 0.3.9", - "winreg 0.6.2", -] - [[package]] name = "ipnet" version = "2.3.0" @@ -1597,9 +1421,9 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jemalloc-sys" @@ -1624,13 +1448,13 @@ dependencies = [ [[package]] name = "jieba-rs" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fbdeee8786790f4a99fa30ff5c5f88aa5183f7583693e3788d17fc8a48f33a" +checksum = "31a7e00da0a90e0de5d8dd1193b0216f5590a24ed633ae701ac50ca94467ef07" dependencies = [ "cedarwood", "fxhash", - "hashbrown 0.9.1", + "hashbrown 0.11.2", "lazy_static", "phf", "phf_codegen", @@ -1639,18 +1463,18 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.46" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" dependencies = [ "wasm-bindgen", ] @@ -1688,9 +1512,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.86" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" +checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" [[package]] name = "linked-hash-map" @@ -1710,10 +1534,28 @@ dependencies = [ ] [[package]] -name = "lock_api" -version = "0.4.2" +name = "local-channel" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +checksum = "6246c68cf195087205a0512559c97e15eaf95198bf0e206d662092cdcb03fe9f" +dependencies = [ + "futures-core", + "futures-sink", + "futures-util", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" + +[[package]] +name = "lock_api" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" dependencies = [ "scopeguard", ] @@ -1728,12 +1570,25 @@ dependencies = [ ] [[package]] -name = "lru-cache" -version = "0.1.2" +name = "logging_timer" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +checksum = "40d0c249955c17c2f8f86b5f501b16d2509ebbe775f7b1d1d2b1ba85ade2a793" dependencies = [ - "linked-hash-map", + "log", + "logging_timer_proc_macros", +] + +[[package]] +name = "logging_timer_proc_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482c2c28e6bcfe7c4274f82f701774d755e6aa873edfd619460fcd0966e0eb07" +dependencies = [ + "log", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", ] [[package]] @@ -1764,17 +1619,17 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" name = "meilisearch-error" version = "0.19.0" dependencies = [ - "actix-http 2.2.0", + "actix-http", ] [[package]] name = "meilisearch-http" -version = "0.17.0" +version = "0.21.0-alpha.2" dependencies = [ "actix-cors", - "actix-http 3.0.0-beta.4", - "actix-rt 2.1.0", - "actix-service 2.0.0-beta.4", + "actix-http", + "actix-rt", + "actix-service", "actix-web", "anyhow", "assert-json-diff", @@ -1787,7 +1642,7 @@ dependencies = [ "crossbeam-channel", "dashmap", "either", - "env_logger 0.8.2", + "env_logger 0.8.3", "flate2", "fst", "futures", @@ -1805,11 +1660,14 @@ dependencies = [ "memmap", "milli", "mime", + "mockall", "once_cell", + "oxidized-json-checker", + "parking_lot", "rand 0.7.3", "rayon", "regex", - "rustls 0.19.0", + "rustls 0.19.1", "sentry", "serde", "serde_json", @@ -1822,9 +1680,27 @@ dependencies = [ "tempdir", "tempfile", "thiserror", - "tokio 1.2.0", + "tokio 1.5.0", + "urlencoding", "uuid", "vergen", + "walkdir", +] + +[[package]] +name = "meilisearch-tokenizer" +version = "0.2.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", ] [[package]] @@ -1845,9 +1721,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d" dependencies = [ "autocfg", ] @@ -1855,11 +1731,12 @@ dependencies = [ [[package]] name = "milli" version = "0.1.1" -source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.1.1#f5ec14c54cc3ab76ca581eca9b6e0870f09fd63f" dependencies = [ "anyhow", "bstr", "byteorder", + "chrono", "crossbeam-channel", "csv", "either", @@ -1873,6 +1750,7 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", + "logging_timer", "meilisearch-tokenizer", "memmap", "num-traits", @@ -1886,9 +1764,11 @@ dependencies = [ "roaring", "serde", "serde_json", + "slice-group-by", "smallstr", "smallvec", "tempfile", + "tinytemplate", "uuid", ] @@ -1910,9 +1790,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", "autocfg", @@ -1939,28 +1819,17 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.9" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" +checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" dependencies = [ "libc", "log", - "miow 0.3.6", + "miow 0.3.7", "ntapi", "winapi 0.3.9", ] -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio 0.6.23", -] - [[package]] name = "miow" version = "0.2.2" @@ -1975,14 +1844,40 @@ dependencies = [ [[package]] name = "miow" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "socket2", "winapi 0.3.9", ] +[[package]] +name = "mockall" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d614ad23f9bb59119b8b5670a85c7ba92c5e9adf4385c81ea00c51c8be33d5" +dependencies = [ + "cfg-if 1.0.0", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd4234635bca06fc96c7368d038061e0aae1b00a764dc817e900dc974e3deea" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", +] + [[package]] name = "net2" version = "0.2.37" @@ -2006,6 +1901,12 @@ dependencies = [ "libc", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "ntapi" version = "0.3.6" @@ -2046,9 +1947,9 @@ dependencies = [ [[package]] name = "object" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" [[package]] name = "obkv" @@ -2058,9 +1959,9 @@ checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" [[package]] name = "once_cell" -version = "1.5.2" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" [[package]] name = "opaque-debug" @@ -2083,6 +1984,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "oxidized-json-checker" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938464aebf563f48ab86d1cfc0e2df952985c0b814d3108f41d1b85e7f5b0dac" + [[package]] name = "page_size" version = "0.4.2" @@ -2106,14 +2013,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "smallvec", "winapi 0.3.9", ] @@ -2159,9 +2066,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "pest_meta", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] @@ -2215,55 +2122,35 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.27" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" dependencies = [ - "pin-project-internal 0.4.27", -] - -[[package]] -name = "pin-project" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" -dependencies = [ - "pin-project-internal 1.0.2", + "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.27" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] name = "pin-project-lite" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" +checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" [[package]] name = "pin-utils" @@ -2283,6 +2170,35 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "predicates" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" + +[[package]] +name = "predicates-tree" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" +dependencies = [ + "predicates-core", + "treeline", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2290,9 +2206,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", "version_check", ] @@ -2302,8 +2218,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.26", + "quote 1.0.9", "version_check", ] @@ -2315,17 +2231,26 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro-nested" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" -version = "1.0.24" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" dependencies = [ - "unicode-xid", + "unicode-xid 0.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +dependencies = [ + "unicode-xid 0.2.1", ] [[package]] @@ -2336,11 +2261,20 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.8" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" dependencies = [ - "proc-macro2", + "proc-macro2 0.4.30", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2 1.0.26", ] [[package]] @@ -2362,7 +2296,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -2423,7 +2357,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", ] [[package]] @@ -2491,7 +2425,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", "lazy_static", "num_cpus", ] @@ -2507,29 +2441,22 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.57" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - -[[package]] -name = "redox_syscall" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.4.3" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -2543,9 +2470,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" [[package]] name = "remove_dir_all" @@ -2578,29 +2505,19 @@ dependencies = [ "mime", "mime_guess", "percent-encoding", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-rustls 0.14.1", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", "webpki-roots 0.20.0", - "winreg 0.7.0", -] - -[[package]] -name = "resolv-conf" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" -dependencies = [ - "hostname", - "quick-error", + "winreg", ] [[package]] @@ -2611,9 +2528,9 @@ checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" [[package]] name = "ring" -version = "0.16.19" +version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ "cc", "libc", @@ -2647,7 +2564,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", ] [[package]] @@ -2665,9 +2591,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ "base64 0.13.0", "log", @@ -2682,6 +2608,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -2690,9 +2625,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "sct" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" dependencies = [ "ring", "untrusted", @@ -2704,7 +2639,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", ] [[package]] @@ -2713,6 +2657,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "sentry" version = "0.18.1" @@ -2731,7 +2684,7 @@ dependencies = [ "rand 0.7.3", "regex", "reqwest", - "rustc_version", + "rustc_version 0.2.3", "sentry-types", "uname", "url", @@ -2754,22 +2707,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.123" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.123" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] @@ -2786,9 +2739,9 @@ dependencies = [ [[package]] name = "serde_url_params" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +checksum = "2c43307d0640738af32fe8d01e47119bc0fc8a686be470a44a586caff76dfb34" dependencies = [ "serde", "url", @@ -2820,9 +2773,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -2839,9 +2792,9 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -2861,9 +2814,9 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" +checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27" [[package]] name = "sized-chunks" @@ -2905,9 +2858,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98d3306e84bf86710d6cd8b4c9c3b721d5454cc91a603180f8f8cd06cfd317b4" +checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a" [[package]] name = "socket2" @@ -2920,6 +2873,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "socket2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "spin" version = "0.5.2" @@ -2928,9 +2891,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.13" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" dependencies = [ "version_check", ] @@ -2942,7 +2905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -2955,11 +2918,11 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.26", + "quote 1.0.9", "serde", "serde_derive", - "syn", + "syn 1.0.69", ] [[package]] @@ -2969,13 +2932,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", - "proc-macro2", - "quote", + "proc-macro2 1.0.26", + "quote 1.0.9", "serde", "serde_derive", "serde_json", "sha1", - "syn", + "syn 1.0.69", ] [[package]] @@ -3009,20 +2972,31 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] name = "syn" -version = "1.0.60" +version = "0.15.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + +[[package]] +name = "syn" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "unicode-xid 0.2.1", ] [[package]] @@ -3040,21 +3014,20 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", + "unicode-xid 0.2.1", ] [[package]] name = "tar" -version = "0.4.30" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +checksum = "c0bcfbd6a598361fda270d82469fff3d65089dc33e175c9a131f7b4cd395f228" dependencies = [ "filetime", "libc", - "redox_syscall 0.1.57", "xattr", ] @@ -3077,7 +3050,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall 0.2.5", + "redox_syscall", "remove_dir_all", "winapi 0.3.9", ] @@ -3115,27 +3088,9 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thread_local" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] @@ -3151,9 +3106,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.23" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +checksum = "08a8cbfbf47955132d0202d1662f49b2423ae35862aee471f3ba4b133358f372" dependencies = [ "const_fn", "libc", @@ -3181,17 +3136,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", + "proc-macro2 1.0.26", + "quote 1.0.9", "standback", - "syn", + "syn 1.0.69", +] + +[[package]] +name = "tinytemplate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" +dependencies = [ + "serde", + "serde_json", ] [[package]] name = "tinyvec" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" dependencies = [ "tinyvec_macros", ] @@ -3204,41 +3169,37 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" dependencies = [ "bytes 0.5.6", "fnv", "futures-core", "iovec", "lazy_static", - "libc", "memchr", "mio 0.6.23", - "mio-uds", "num_cpus", - "pin-project-lite 0.1.11", - "signal-hook-registry", + "pin-project-lite 0.1.12", "slab", - "winapi 0.3.9", ] [[package]] name = "tokio" -version = "1.2.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" dependencies = [ "autocfg", "bytes 1.0.1", "libc", "memchr", - "mio 0.7.9", + "mio 0.7.11", "num_cpus", "once_cell", "parking_lot", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "signal-hook-registry", "tokio-macros", "winapi 0.3.9", @@ -3250,9 +3211,9 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", ] [[package]] @@ -3263,7 +3224,7 @@ checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", "rustls 0.18.1", - "tokio 0.2.24", + "tokio 0.2.25", "webpki", ] @@ -3273,8 +3234,8 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ - "rustls 0.19.0", - "tokio 1.2.0", + "rustls 0.19.1", + "tokio 1.5.0", "webpki", ] @@ -3288,39 +3249,39 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.1.11", - "tokio 0.2.24", + "pin-project-lite 0.1.12", + "tokio 0.2.25", ] [[package]] name = "tokio-util" -version = "0.6.3" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +checksum = "940a12c99365c31ea8dd9ba04ec1be183ffe4920102bb7122c2f515437601e8e" dependencies = [ "bytes 1.0.1", "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.0", - "tokio 1.2.0", + "pin-project-lite 0.2.6", + "tokio 1.5.0", ] [[package]] name = "tower-service" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.22" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "tracing-core", ] @@ -3335,53 +3296,19 @@ dependencies = [ [[package]] name = "tracing-futures" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 0.4.27", + "pin-project", "tracing", ] [[package]] -name = "trust-dns-proto" -version = "0.19.6" +name = "treeline" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" -dependencies = [ - "async-trait", - "backtrace", - "enum-as-inner", - "futures", - "idna", - "lazy_static", - "log", - "rand 0.7.3", - "smallvec", - "thiserror", - "tokio 0.2.24", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.19.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" -dependencies = [ - "backtrace", - "cfg-if 0.1.10", - "futures", - "ipconfig", - "lazy_static", - "log", - "lru-cache", - "resolv-conf", - "smallvec", - "thiserror", - "tokio 0.2.24", - "trust-dns-proto", -] +checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "try-lock" @@ -3391,9 +3318,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" [[package]] name = "ucd-trie" @@ -3421,18 +3348,18 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" dependencies = [ "matches", ] [[package]] name = "unicode-normalization" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" dependencies = [ "tinyvec", ] @@ -3449,6 +3376,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.1" @@ -3463,9 +3396,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" dependencies = [ "form_urlencoded", "idna", @@ -3474,6 +3407,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" + [[package]] name = "utf8-width" version = "0.1.4" @@ -3498,19 +3437,31 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vergen" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" dependencies = [ "bitflags", "chrono", + "rustc_version 0.3.3", ] [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi 0.3.9", + "winapi-util", +] [[package]] name = "want" @@ -3536,9 +3487,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.69" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" dependencies = [ "cfg-if 1.0.0", "serde", @@ -3548,24 +3499,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.69" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.19" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3575,38 +3526,38 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.69" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" dependencies = [ - "quote", + "quote 1.0.9", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.69" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.69" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" +checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" [[package]] name = "web-sys" -version = "0.3.46" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" dependencies = [ "js-sys", "wasm-bindgen", @@ -3633,9 +3584,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" dependencies = [ "webpki", ] @@ -3649,12 +3600,6 @@ dependencies = [ "hashbrown 0.7.2", ] -[[package]] -name = "widestring" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" - [[package]] name = "winapi" version = "0.2.8" @@ -3698,15 +3643,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "winreg" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "winreg" version = "0.7.0" @@ -3747,12 +3683,12 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" +checksum = "dc9c39e6d503229ffa00cc2954af4a751e6bbedf2a2c18e856eb3ece93d32495" dependencies = [ - "proc-macro2", - "syn", + "proc-macro2 1.0.26", + "syn 1.0.69", "synstructure", ] diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index f8835eceb..ab5fc6fb5 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -59,6 +59,9 @@ impl Index { }) .transpose()? .unwrap_or_else(BTreeSet::new); + let distinct_attribute = self + .distinct_attribute(&txn)? + .map(String::from); Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), @@ -66,6 +69,7 @@ impl Index { attributes_for_faceting: Some(Some(faceted_attributes)), ranking_rules: Some(Some(criteria)), stop_words: Some(Some(stop_words)), + distinct_attribute: Some(distinct_attribute), }) } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 085115af6..f91da257f 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -43,13 +43,18 @@ pub struct Settings { skip_serializing_if = "Option::is_none" )] pub ranking_rules: Option>>, - #[serde( default, deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none" )] pub stop_words: Option>>, + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] + pub distinct_attribute: Option>, } impl Settings { @@ -60,6 +65,7 @@ impl Settings { attributes_for_faceting: Some(None), ranking_rules: Some(None), stop_words: Some(None), + distinct_attribute: Some(None), } } } @@ -145,7 +151,6 @@ impl Index { let mut wtxn = self.write_txn()?; let mut builder = update_builder.settings(&mut wtxn, self); - // We transpose the settings JSON struct into a real setting update. if let Some(ref names) = settings.searchable_attributes { match names { Some(names) => builder.set_searchable_fields(names.clone()), @@ -153,7 +158,6 @@ impl Index { } } - // We transpose the settings JSON struct into a real setting update. if let Some(ref names) = settings.displayed_attributes { match names { Some(names) => builder.set_displayed_fields(names.clone()), @@ -161,13 +165,11 @@ impl Index { } } - // We transpose the settings JSON struct into a real setting update. if let Some(ref facet_types) = settings.attributes_for_faceting { let facet_types = facet_types.clone().unwrap_or_else(HashMap::new); builder.set_faceted_fields(facet_types); } - // We transpose the settings JSON struct into a real setting update. if let Some(ref criteria) = settings.ranking_rules { match criteria { Some(criteria) => builder.set_criteria(criteria.clone()), @@ -175,7 +177,6 @@ impl Index { } } - // We transpose the settings JSON struct into a real setting update. if let Some(ref stop_words) = settings.stop_words { match stop_words { Some(stop_words) => builder.set_stop_words(stop_words.clone()), @@ -183,6 +184,13 @@ impl Index { } } + if let Some(ref distinct_attribute) = settings.distinct_attribute { + match distinct_attribute { + Some(attr) => builder.set_distinct_attribute(attr.clone()), + None => builder.reset_distinct_attribute(), + } + } + let result = builder .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); From 1746132c7dd40b7534e042ce89e64975ade2b5e0 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 6 Apr 2021 17:56:25 +0200 Subject: [PATCH 260/522] add test set/reset distinct attribute --- meilisearch-http/tests/settings/distinct.rs | 23 +++++++++++++++++++ .../tests/settings/get_settings.rs | 1 + meilisearch-http/tests/settings/mod.rs | 1 + 3 files changed, 25 insertions(+) create mode 100644 meilisearch-http/tests/settings/distinct.rs diff --git a/meilisearch-http/tests/settings/distinct.rs b/meilisearch-http/tests/settings/distinct.rs new file mode 100644 index 000000000..852276b99 --- /dev/null +++ b/meilisearch-http/tests/settings/distinct.rs @@ -0,0 +1,23 @@ +use crate::common::Server; +use serde_json::{json, Value}; + +#[actix_rt::test] +async fn set_and_reset_distinct_attribute() { + let server = Server::new().await; + let index = server.index("test"); + + let (_response, _code) = index.update_settings(json!({ "distinctAttribute": "test"})).await; + index.wait_update_id(0).await; + + let (response, _) = index.settings().await; + + assert_eq!(response["distinctAttribute"], "test"); + + index.update_settings(json!({ "distinctAttribute": Value::Null })).await; + + index.wait_update_id(1).await; + + let (response, _) = index.settings().await; + + assert_eq!(response["distinctAttribute"], Value::Null); +} diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index be09a5090..d83a778cc 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -20,6 +20,7 @@ async fn get_settings() { assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["attributesForFaceting"], json!({})); + assert_eq!(settings["distinctAttribute"], serde_json::Value::Null); assert_eq!( settings["rankingRules"], json!([ diff --git a/meilisearch-http/tests/settings/mod.rs b/meilisearch-http/tests/settings/mod.rs index c9e93c85d..b7102cc5f 100644 --- a/meilisearch-http/tests/settings/mod.rs +++ b/meilisearch-http/tests/settings/mod.rs @@ -1 +1,2 @@ mod get_settings; +mod distinct; From ec63e138968802b045b5411b926f3f7754a7dafa Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 19 Apr 2021 20:23:25 +0200 Subject: [PATCH 261/522] bump actix --- meilisearch-http/tests/settings/get_settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index d83a778cc..82be6d994 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -16,7 +16,7 @@ async fn get_settings() { let (response, code) = index.settings().await; assert_eq!(code, 200); let settings = response.as_object().unwrap(); - assert_eq!(settings.keys().len(), 5); + assert_eq!(settings.keys().len(), 6); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["attributesForFaceting"], json!({})); From 6c470cf6876b60c74d8025f5551847a79e2ef06e Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 20 Apr 2021 11:28:16 +0200 Subject: [PATCH 262/522] enable distinct-attribute setting route --- meilisearch-http/src/routes/settings/mod.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 732888ec2..54cc53d7e 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -97,11 +97,11 @@ make_setting_route!( stop_words ); -//make_setting_route!( -//"/indexes/{index_uid}/settings/distinct-attribute", -//String, -//distinct_attribute -//); +make_setting_route!( + "/indexes/{index_uid}/settings/distinct-attribute", + String, + distinct_attribute +); //make_setting_route!( //"/indexes/{index_uid}/settings/ranking-rules", @@ -129,6 +129,7 @@ create_services!( attributes_for_faceting, displayed_attributes, searchable_attributes, + distinct_attribute, stop_words ); From f8c338e3a7dad9b88062f02000dff60902899af1 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 20 Apr 2021 12:07:22 +0200 Subject: [PATCH 263/522] add test for dedicated distinct route --- Cargo.lock | 7 +++++ meilisearch-http/Cargo.toml | 1 + meilisearch-http/tests/common/index.rs | 23 ++++++++++++++++ meilisearch-http/tests/settings/distinct.rs | 27 ++++++++++++++++--- .../tests/settings/get_settings.rs | 2 +- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5baa6d8d..df129df98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1664,6 +1664,7 @@ dependencies = [ "once_cell", "oxidized-json-checker", "parking_lot", + "paste", "rand 0.7.3", "rayon", "regex", @@ -2025,6 +2026,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "paste" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" + [[package]] name = "percent-encoding" version = "2.1.0" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e1d46c8e2..8532932e4 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -84,6 +84,7 @@ version = "0.18.1" actix-rt = "2.1.0" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } mockall = "0.9.1" +paste = "1.0.5" serde_url_params = "0.2.0" tempdir = "0.3.7" urlencoding = "1.1.1" diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 67ea6c19a..aaf108cf6 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -1,16 +1,34 @@ use std::time::Duration; use actix_web::http::StatusCode; +use paste::paste; use serde_json::{json, Value}; use tokio::time::sleep; use super::service::Service; +macro_rules! make_settings_test_routes { + ($($name:ident),+) => { + $(paste! { + pub async fn [](&self, value: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/{}", self.uid, stringify!($name).replace("_", "-")); + self.service.post(url, value).await + } + + pub async fn [](&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/{}", self.uid, stringify!($name).replace("_", "-")); + self.service.get(url).await + } + })* + }; +} + pub struct Index<'a> { pub uid: String, pub service: &'a Service, } +#[allow(dead_code)] impl Index<'_> { pub async fn get(&self) -> (Value, StatusCode) { let url = format!("/indexes/{}", self.uid); @@ -166,8 +184,13 @@ impl Index<'_> { let url = format!("/indexes/{}/stats", self.uid); self.service.get(url).await } + + make_settings_test_routes!( + distinct_attribute + ); } + pub struct GetDocumentOptions; #[derive(Debug, Default)] diff --git a/meilisearch-http/tests/settings/distinct.rs b/meilisearch-http/tests/settings/distinct.rs index 852276b99..a3aec6baf 100644 --- a/meilisearch-http/tests/settings/distinct.rs +++ b/meilisearch-http/tests/settings/distinct.rs @@ -1,5 +1,5 @@ use crate::common::Server; -use serde_json::{json, Value}; +use serde_json::json; #[actix_rt::test] async fn set_and_reset_distinct_attribute() { @@ -13,11 +13,32 @@ async fn set_and_reset_distinct_attribute() { assert_eq!(response["distinctAttribute"], "test"); - index.update_settings(json!({ "distinctAttribute": Value::Null })).await; + index.update_settings(json!({ "distinctAttribute": null })).await; index.wait_update_id(1).await; let (response, _) = index.settings().await; - assert_eq!(response["distinctAttribute"], Value::Null); + assert_eq!(response["distinctAttribute"], json!(null)); +} + +#[actix_rt::test] +async fn set_and_reset_distinct_attribute_with_dedicated_route() { + let server = Server::new().await; + let index = server.index("test"); + + let (_response, _code) = index.update_distinct_attribute(json!("test")).await; + index.wait_update_id(0).await; + + let (response, _) = index.get_distinct_attribute().await; + + assert_eq!(response, "test"); + + index.update_distinct_attribute(json!(null)).await; + + index.wait_update_id(1).await; + + let (response, _) = index.get_distinct_attribute().await; + + assert_eq!(response, json!(null)); } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 82be6d994..3412f45af 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -20,7 +20,7 @@ async fn get_settings() { assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["attributesForFaceting"], json!({})); - assert_eq!(settings["distinctAttribute"], serde_json::Value::Null); + assert_eq!(settings["distinctAttribute"], json!(null)); assert_eq!( settings["rankingRules"], json!([ From b8e535579f5f14b867f4514499902f0d2ebf094f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 20 Apr 2021 16:11:07 +0200 Subject: [PATCH 264/522] Update version for the next release (alpha3) --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df129df98..79e013392 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1624,7 +1624,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.21.0-alpha.2" +version = "0.21.0-alpha.3" dependencies = [ "actix-cors", "actix-http", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 8532932e4..5ba3acd2b 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "MeiliSearch HTTP server" edition = "2018" license = "MIT" name = "meilisearch-http" -version = "0.21.0-alpha.2" +version = "0.21.0-alpha.3" [[bin]] name = "meilisearch" path = "src/main.rs" From 526a05565efbdf9fadbaeee8267d4ac4a6351b65 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 19 Apr 2021 10:13:13 +0200 Subject: [PATCH 265/522] add SearchHit structure --- meilisearch-http/src/index/search.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 4b9753b82..28eb7af98 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -35,10 +35,18 @@ pub struct SearchQuery { pub facet_distributions: Option>, } +#[derive(Debug, Clone, Serialize)] +pub struct SearchHit { + #[serde(flatten)] + pub document: Map, + #[serde(rename = "_formatted", skip_serializing_if = "Map::is_empty")] + pub formatted: Map, +} + #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct SearchResult { - pub hits: Vec>, + pub hits: Vec, pub nb_hits: u64, pub exhaustive_nb_hits: bool, pub query: String, @@ -90,7 +98,11 @@ impl Index { if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } - documents.push(object); + let hit = SearchHit { + document: object, + formatted: Map::new(), + }; + documents.push(hit); } let nb_hits = candidates.len(); From c6bb36efa5bdf689fab0985d46c47862722ec2cb Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 19 Apr 2021 16:22:41 +0200 Subject: [PATCH 266/522] implement _formated --- Cargo.lock | 9 +- meilisearch-http/Cargo.toml | 1 + meilisearch-http/src/index/mod.rs | 6 +- meilisearch-http/src/index/search.rs | 148 ++++++++++++++++++++------ meilisearch-http/src/routes/search.rs | 4 +- 5 files changed, 124 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79e013392..1e5809b17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,9 +345,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" +checksum = "78ed203b9ba68b242c62b3fb7480f589dd49829be1edb3fe8fc8b4ffda2dcb8d" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -1661,6 +1661,7 @@ dependencies = [ "milli", "mime", "mockall", + "obkv", "once_cell", "oxidized-json-checker", "parking_lot", @@ -2837,9 +2838,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" [[package]] name = "slice-group-by" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 5ba3acd2b..e7a55d28e 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -63,6 +63,7 @@ tokio = { version = "1", features = ["full"] } uuid = "0.8.2" oxidized-json-checker = "0.3.2" walkdir = "2.3.2" +obkv = "0.1.1" [dependencies.sentry] default-features = false diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index ab5fc6fb5..57a939aaa 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -83,7 +83,7 @@ impl Index { let fields_ids_map = self.fields_ids_map(&txn)?; let fields_to_display = - self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; + self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -108,7 +108,7 @@ impl Index { let fields_ids_map = self.fields_ids_map(&txn)?; let fields_to_display = - self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; + self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; let internal_id = self .external_documents_ids(&txn)? @@ -134,7 +134,7 @@ impl Index { fn fields_to_display>( &self, txn: &heed::RoTxn, - attributes_to_retrieve: Option>, + attributes_to_retrieve: &Option>, fields_ids_map: &milli::FieldsIdsMap, ) -> anyhow::Result> { let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 28eb7af98..9e0c5a5cc 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,17 +1,19 @@ +use std::borrow::Cow; use std::collections::{BTreeMap, HashSet}; -use std::mem; use std::time::Instant; use anyhow::bail; use either::Either; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{facet::FacetValue, FacetCondition, MatchingWords}; +use milli::{facet::FacetValue, FacetCondition, FieldId, FieldsIdsMap, MatchingWords}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use super::Index; +pub type Document = Map; + pub const DEFAULT_SEARCH_LIMIT: usize = 20; const fn default_search_limit() -> usize { @@ -25,8 +27,8 @@ pub struct SearchQuery { pub offset: Option, #[serde(default = "default_search_limit")] pub limit: usize, - pub attributes_to_retrieve: Option>, - pub attributes_to_crop: Option>, + pub attributes_to_retrieve: Option>, + pub attributes_to_crop: Option>, pub crop_length: Option, pub attributes_to_highlight: Option>, pub filters: Option, @@ -38,9 +40,9 @@ pub struct SearchQuery { #[derive(Debug, Clone, Serialize)] pub struct SearchHit { #[serde(flatten)] - pub document: Map, + pub document: Document, #[serde(rename = "_formatted", skip_serializing_if = "Map::is_empty")] - pub formatted: Map, + pub formatted: Document, } #[derive(Serialize)] @@ -86,21 +88,79 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let fields_to_display = - self.fields_to_display(&rtxn, query.attributes_to_retrieve, &fields_ids_map)?; + let fids = |attrs: &HashSet| { + attrs + .iter() + .filter_map(|name| fields_ids_map.id(name)) + .collect() + }; + + let displayed_ids: HashSet = fields_ids_map.iter().map(|(id, _)| id).collect(); + + let to_retrieve_ids = query + .attributes_to_retrieve + .as_ref() + .map(fids) + .unwrap_or_else(|| displayed_ids.clone()); + + let to_highlight_ids = query + .attributes_to_highlight + .as_ref() + .map(fids) + .unwrap_or_default(); + + let to_crop_ids = query + .attributes_to_crop + .as_ref() + .map(fids) + .unwrap_or_default(); + + // The attributes to retrieve are: + // - the ones explicitly marked as to retrieve that are also in the displayed attributes + let all_attributes: Vec<_> = to_retrieve_ids + .intersection(&displayed_ids) + .cloned() + .collect(); + + // The formatted attributes are: + // - The one in either highlighted attributes or cropped attributes if there are attributes + // to retrieve + // - All the attributes to retrieve if there are either highlighted or cropped attributes + // the request specified that all attributes are to retrieve (i.e attributes to retrieve is + // empty in the query) + let all_formatted = if query.attributes_to_retrieve.is_none() { + if query.attributes_to_highlight.is_some() || query.attributes_to_crop.is_some() { + Cow::Borrowed(&all_attributes) + } else { + Cow::Owned(Vec::new()) + } + } else { + let attrs = (&to_crop_ids | &to_highlight_ids) + .intersection(&displayed_ids) + .cloned() + .collect::>(); + Cow::Owned(attrs) + }; let stop_words = fst::Set::default(); - let highlighter = Highlighter::new(&stop_words); + let highlighter = Highlighter::new( + &stop_words, + (String::from(""), String::from("")), + ); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let mut object = - milli::obkv_to_json(&fields_to_display, &fields_ids_map, obkv).unwrap(); - if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { - highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); - } + let document = milli::obkv_to_json(&all_attributes, &fields_ids_map, obkv.clone())?; + let formatted = compute_formatted( + &fields_ids_map, + obkv, + &highlighter, + &matching_words, + all_formatted.as_ref().as_slice(), + &to_highlight_ids, + )?; let hit = SearchHit { - document: object, - formatted: Map::new(), + document, + formatted, }; documents.push(hit); } @@ -132,6 +192,38 @@ impl Index { } } +fn compute_formatted>( + field_ids_map: &FieldsIdsMap, + obkv: obkv::KvReader, + highlighter: &Highlighter, + matching_words: &MatchingWords, + all_formatted: &[FieldId], + to_highlight_ids: &HashSet, +) -> anyhow::Result { + let mut document = Document::new(); + + for field in all_formatted { + if let Some(value) = obkv.get(*field) { + let mut value: Value = serde_json::from_slice(value)?; + + if to_highlight_ids.contains(field) { + value = highlighter.highlight_value(value, matching_words); + } + + // This unwrap must be safe since we got the ids from the fields_ids_map just + // before. + let key = field_ids_map + .name(*field) + .expect("Missing field name") + .to_string(); + + document.insert(key, value); + } + } + + Ok(document) +} + fn parse_facets_array( txn: &RoTxn, index: &Index, @@ -163,16 +255,17 @@ fn parse_facets_array( pub struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, + marks: (String, String), } impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { - pub fn new(stop_words: &'a fst::Set) -> Self { + pub fn new(stop_words: &'a fst::Set, marks: (String, String)) -> Self { let mut config = AnalyzerConfig::default(); config.stop_words(stop_words); let analyzer = Analyzer::new(config); - Self { analyzer } + Self { analyzer, marks } } pub fn highlight_value(&self, value: Value, words_to_highlight: &MatchingWords) -> Value { @@ -187,11 +280,11 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { if token.is_word() { let to_highlight = words_to_highlight.matches(token.text()); if to_highlight { - string.push_str("") + string.push_str(&self.marks.0) } string.push_str(word); if to_highlight { - string.push_str("") + string.push_str(&self.marks.1) } } else { string.push_str(word); @@ -213,21 +306,6 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { ), } } - - pub fn highlight_record( - &self, - object: &mut Map, - words_to_highlight: &MatchingWords, - attributes_to_highlight: &HashSet, - ) { - // TODO do we need to create a string for element that are not and needs to be highlight? - for (key, value) in object.iter_mut() { - if attributes_to_highlight.contains(key) { - let old_value = mem::take(value); - *value = self.highlight_value(old_value, words_to_highlight); - } - } - } } fn parse_facets( diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index ae2caeb30..86beb2750 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -36,11 +36,11 @@ impl TryFrom for SearchQuery { fn try_from(other: SearchQueryGet) -> anyhow::Result { let attributes_to_retrieve = other .attributes_to_retrieve - .map(|attrs| attrs.split(',').map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let attributes_to_crop = other .attributes_to_crop - .map(|attrs| attrs.split(',').map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let attributes_to_highlight = other .attributes_to_highlight From 881b099c8edd6273f26c6dae3d603bf5aad34101 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 19 Apr 2021 19:03:53 +0200 Subject: [PATCH 267/522] add tests --- meilisearch-http/src/index/search.rs | 137 ++++++++++++++++++++++++++- meilisearch-http/src/routes/index.rs | 13 +-- 2 files changed, 135 insertions(+), 15 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 9e0c5a5cc..bc9554864 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -196,7 +196,7 @@ fn compute_formatted>( field_ids_map: &FieldsIdsMap, obkv: obkv::KvReader, highlighter: &Highlighter, - matching_words: &MatchingWords, + matching_words: &impl Matcher, all_formatted: &[FieldId], to_highlight_ids: &HashSet, ) -> anyhow::Result { @@ -224,6 +224,24 @@ fn compute_formatted>( Ok(document) } +/// trait to allow unit testing of `compute_formated` +trait Matcher { + fn matches(&self, w: &str) -> bool; +} + +#[cfg(test)] +impl Matcher for HashSet { + fn matches(&self, w: &str) -> bool { + self.contains(w) + } +} + +impl Matcher for MatchingWords { + fn matches(&self, w: &str) -> bool { + self.matches(w) + } +} + fn parse_facets_array( txn: &RoTxn, index: &Index, @@ -253,7 +271,7 @@ fn parse_facets_array( FacetCondition::from_array(txn, &index.0, ands) } -pub struct Highlighter<'a, A> { +struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, marks: (String, String), } @@ -268,7 +286,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { Self { analyzer, marks } } - pub fn highlight_value(&self, value: Value, words_to_highlight: &MatchingWords) -> Value { + fn highlight_value(&self, value: Value, words_to_highlight: &impl Matcher) -> Value { match value { Value::Null => Value::Null, Value::Bool(boolean) => Value::Bool(boolean), @@ -320,3 +338,116 @@ fn parse_facets( v => bail!("Invalid facet expression, expected Array, found: {:?}", v), } } + +#[cfg(test)] +mod test { + use std::iter::FromIterator; + use serde_json::json; + + use super::*; + + #[test] + fn no_formatted() { + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new( + &stop_words, + (String::from(""), String::from("")), + ); + + let mut fields = FieldsIdsMap::new(); + let id = fields.insert("test").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let all_formatted = Vec::new(); + let to_highlight_ids = HashSet::new(); + + let matching_words = MatchingWords::default(); + + let value = compute_formatted( + &fields, + obkv, + &highlighter, + &matching_words, + &all_formatted, + &to_highlight_ids + ).unwrap(); + + assert!(value.is_empty()); + } + + #[test] + fn formatted_no_highlight() { + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new( + &stop_words, + (String::from(""), String::from("")), + ); + + let mut fields = FieldsIdsMap::new(); + let id = fields.insert("test").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let all_formatted = vec![id]; + let to_highlight_ids = HashSet::new(); + + let matching_words = MatchingWords::default(); + + let value = compute_formatted( + &fields, + obkv, + &highlighter, + &matching_words, + &all_formatted, + &to_highlight_ids + ).unwrap(); + + assert_eq!(Value::Object(value), json!({"test": "hello"})); + } + + #[test] + fn formatted_with_highlight() { + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new( + &stop_words, + (String::from(""), String::from("")), + ); + + let mut fields = FieldsIdsMap::new(); + let id = fields.insert("test").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let all_formatted = vec![id]; + let to_highlight_ids = HashSet::from_iter(Some(id)); + + let matching_words = HashSet::from_iter(Some(String::from("hello"))); + + let value = compute_formatted( + &fields, + obkv, + &highlighter, + &matching_words, + &all_formatted, + &to_highlight_ids + ).unwrap(); + + assert_eq!(Value::Object(value), json!({"test": "hello"})); + } +} diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index d5fac5170..4424c8cfe 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -1,7 +1,6 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use crate::error::ResponseError; use crate::helpers::Authentication; @@ -69,16 +68,6 @@ struct UpdateIndexRequest { primary_key: Option, } -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct UpdateIndexResponse { - name: String, - uid: String, - created_at: DateTime, - updated_at: DateTime, - primary_key: Option, -} - #[put("/indexes/{index_uid}", wrap = "Authentication::Private")] async fn update_index( data: web::Data, From 7a737d2bd35c01049527c30f16ae9423cd51ab93 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 20 Apr 2021 13:10:50 +0200 Subject: [PATCH 268/522] support wildcard --- meilisearch-http/src/index/search.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index bc9554864..4f796476c 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -41,7 +41,7 @@ pub struct SearchQuery { pub struct SearchHit { #[serde(flatten)] pub document: Document, - #[serde(rename = "_formatted", skip_serializing_if = "Map::is_empty")] + #[serde(rename = "_formatted", skip_serializing_if = "Document::is_empty")] pub formatted: Document, } @@ -88,15 +88,23 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let fids = |attrs: &HashSet| { - attrs - .iter() - .filter_map(|name| fields_ids_map.id(name)) - .collect() - }; - let displayed_ids: HashSet = fields_ids_map.iter().map(|(id, _)| id).collect(); + let fids = |attrs: &HashSet| { + let mut ids = HashSet::new(); + for attr in attrs { + if attr == "*" { + ids = displayed_ids.clone(); + break; + } + + if let Some(id) = fields_ids_map.id(attr) { + ids.insert(id); + } + } + ids + }; + let to_retrieve_ids = query .attributes_to_retrieve .as_ref() From d9a29cae608e68f1b1128e96dfe139f64e08052b Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 20 Apr 2021 16:21:30 +0200 Subject: [PATCH 269/522] fix ignored displayed attributes --- meilisearch-http/src/index/search.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 4f796476c..2af626875 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -88,7 +88,9 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let displayed_ids: HashSet = fields_ids_map.iter().map(|(id, _)| id).collect(); + let displayed_ids: HashSet = self.displayed_fields_ids(&rtxn)? + .map(|fields| fields.into_iter().collect::>()) + .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect()); let fids = |attrs: &HashSet| { let mut ids = HashSet::new(); From dd2914873b77843ee71eac36d3a3cbd7c40ab991 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 20 Apr 2021 21:19:37 +0200 Subject: [PATCH 270/522] fix document fields order --- meilisearch-http/src/index/search.rs | 39 +++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 2af626875..63e8b7cd9 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -5,14 +5,16 @@ use std::time::Instant; use anyhow::bail; use either::Either; use heed::RoTxn; +use indexmap::IndexMap; +use itertools::Itertools; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; use milli::{facet::FacetValue, FacetCondition, FieldId, FieldsIdsMap, MatchingWords}; use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; +use serde_json::Value; use super::Index; -pub type Document = Map; +pub type Document = IndexMap; pub const DEFAULT_SEARCH_LIMIT: usize = 20; @@ -88,7 +90,7 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let displayed_ids: HashSet = self.displayed_fields_ids(&rtxn)? + let displayed_ids = self.displayed_fields_ids(&rtxn)? .map(|fields| fields.into_iter().collect::>()) .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect()); @@ -130,6 +132,7 @@ impl Index { let all_attributes: Vec<_> = to_retrieve_ids .intersection(&displayed_ids) .cloned() + .sorted() .collect(); // The formatted attributes are: @@ -159,7 +162,7 @@ impl Index { ); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let document = milli::obkv_to_json(&all_attributes, &fields_ids_map, obkv.clone())?; + let document = make_document(&all_attributes, &fields_ids_map, obkv.clone())?; let formatted = compute_formatted( &fields_ids_map, obkv, @@ -202,6 +205,29 @@ impl Index { } } +fn make_document( + attributes_to_retrieve: &[FieldId], + field_ids_map: &FieldsIdsMap, + obkv: obkv::KvReader, +) -> anyhow::Result { + let mut document = Document::new(); + for attr in attributes_to_retrieve { + if let Some(value) = obkv.get(*attr) { + let value = serde_json::from_slice(value)?; + + // This unwrap must be safe since we got the ids from the fields_ids_map just + // before. + let key = field_ids_map + .name(*attr) + .expect("Missing field name") + .to_string(); + + document.insert(key, value); + } + } + Ok(document) +} + fn compute_formatted>( field_ids_map: &FieldsIdsMap, obkv: obkv::KvReader, @@ -352,7 +378,6 @@ fn parse_facets( #[cfg(test)] mod test { use std::iter::FromIterator; - use serde_json::json; use super::*; @@ -423,7 +448,7 @@ mod test { &to_highlight_ids ).unwrap(); - assert_eq!(Value::Object(value), json!({"test": "hello"})); + assert_eq!(value["test"], "hello"); } #[test] @@ -458,6 +483,6 @@ mod test { &to_highlight_ids ).unwrap(); - assert_eq!(Value::Object(value), json!({"test": "hello"})); + assert_eq!(value["test"], "hello"); } } From ce5e4743e6e6b68ce5a3b13d8c00ba9db0d258d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 21 Apr 2021 11:00:04 +0200 Subject: [PATCH 271/522] Fix dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bbaa4a570..8c2648512 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ COPY Cargo.lock . COPY Cargo.toml . COPY meilisearch-error/Cargo.toml meilisearch-error/ -COPY meilisearch-http/Cargo.lock meilisearch-http/ COPY meilisearch-http/Cargo.toml meilisearch-http/ ENV RUSTFLAGS="-C target-feature=-crt-static" From a72d2f66cd2cfe7e41058ded7b84274e09bdf353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 21 Apr 2021 19:14:55 +0200 Subject: [PATCH 272/522] use tags instead of tags for highlighting --- meilisearch-http/src/index/search.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 63e8b7cd9..1cccfcd58 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -158,7 +158,7 @@ impl Index { let stop_words = fst::Set::default(); let highlighter = Highlighter::new( &stop_words, - (String::from(""), String::from("")), + (String::from(""), String::from("")), ); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { @@ -386,7 +386,7 @@ mod test { let stop_words = fst::Set::default(); let highlighter = Highlighter::new( &stop_words, - (String::from(""), String::from("")), + (String::from(""), String::from("")), ); let mut fields = FieldsIdsMap::new(); @@ -421,7 +421,7 @@ mod test { let stop_words = fst::Set::default(); let highlighter = Highlighter::new( &stop_words, - (String::from(""), String::from("")), + (String::from(""), String::from("")), ); let mut fields = FieldsIdsMap::new(); @@ -456,7 +456,7 @@ mod test { let stop_words = fst::Set::default(); let highlighter = Highlighter::new( &stop_words, - (String::from(""), String::from("")), + (String::from(""), String::from("")), ); let mut fields = FieldsIdsMap::new(); @@ -483,6 +483,6 @@ mod test { &to_highlight_ids ).unwrap(); - assert_eq!(value["test"], "hello"); + assert_eq!(value["test"], "hello"); } } From ea5517bc8c7658f232d1d1f22f7f504a243c8eee Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 20 Apr 2021 13:28:12 +0200 Subject: [PATCH 273/522] add mini-dashboard feature --- Cargo.lock | 68 ++++++++++++++++++------------------- meilisearch-http/Cargo.toml | 1 + meilisearch-http/build.rs | 6 ++++ 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e5809b17..2ae1cc8d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "actix-codec" -version = "0.4.0-beta.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e" +checksum = "1d5dbeb2d9e51344cb83ca7cc170f1217f9fe25bfc50160e6e200b5c31c1019a" dependencies = [ "bitflags", "bytes 1.0.1", @@ -84,7 +84,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -213,7 +213,7 @@ checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +checksum = "0a26cb53174ddd320edfff199a853f93d571f48eeb4dde75e67a9a3dbb7b7e5e" dependencies = [ "async-stream-impl", "futures-core", @@ -306,13 +306,13 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -323,7 +323,7 @@ checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -475,9 +475,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byte-unit" -version = "4.0.10" +version = "4.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9520900471c3a9bbcfe0fd4c7b6bcfeff41b20a76cf91c59b7474b09be1ee27" +checksum = "26d98e67b09c0321733bef2f3b879832afa6197e9ea58f32e72c316df2ffe743" dependencies = [ "utf8-width", ] @@ -748,7 +748,7 @@ dependencies = [ "convert_case", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -852,7 +852,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", "synstructure", ] @@ -1008,7 +1008,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -1877,7 +1877,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -2076,7 +2076,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -2145,7 +2145,7 @@ checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -2216,7 +2216,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", "version_check", ] @@ -2551,9 +2551,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6744a4a918e91359ad1d356a91e2e943a86d9fb9ae77f715d617032ea2af88f" +checksum = "a4b2e7ab0bbb2d144558ae3f4761a0db06d21463b45756fc64c3393cdba3d447" dependencies = [ "bytemuck", "byteorder", @@ -2730,7 +2730,7 @@ checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -2930,7 +2930,7 @@ dependencies = [ "quote 1.0.9", "serde", "serde_derive", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -2946,7 +2946,7 @@ dependencies = [ "serde_derive", "serde_json", "sha1", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -2982,7 +2982,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -2998,9 +2998,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +checksum = "b9505f307c872bab8eb46f77ae357c8eba1fdacead58ee5a850116b1d7f82883" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", @@ -3024,7 +3024,7 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", "unicode-xid 0.2.1", ] @@ -3098,7 +3098,7 @@ checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -3147,7 +3147,7 @@ dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", "standback", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -3221,7 +3221,7 @@ checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", ] [[package]] @@ -3516,7 +3516,7 @@ dependencies = [ "log", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", "wasm-bindgen-shared", ] @@ -3550,7 +3550,7 @@ checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.69", + "syn 1.0.70", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3696,7 +3696,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc9c39e6d503229ffa00cc2954af4a751e6bbedf2a2c18e856eb3ece93d32495" dependencies = [ "proc-macro2 1.0.26", - "syn 1.0.69", + "syn 1.0.70", "synstructure", ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e7a55d28e..e45311fa0 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -91,6 +91,7 @@ tempdir = "0.3.7" urlencoding = "1.1.1" [features] +mini-dashboard = ["default"] default = ["sentry"] [target.'cfg(target_os = "linux")'.dependencies] diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index 2257407a8..be4f4a1b6 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -1,3 +1,5 @@ +use std::env; + use vergen::{generate_cargo_keys, ConstantsFlags}; fn main() { @@ -7,4 +9,8 @@ fn main() { // Generate the 'cargo:' key output generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!"); + + if let Ok(_) = env::var("CARGO_FEATURE_MINI_DASHBOARD") { + todo!() + } } From bb79695e44d6e1f2a4d7a27b41d8c966ac9f0e76 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 20 Apr 2021 15:20:09 +0200 Subject: [PATCH 274/522] load mini-dashboard assets --- Cargo.lock | 163 ++++++++++++++++++++++++++++++++++-- meilisearch-http/Cargo.toml | 10 +++ meilisearch-http/build.rs | 44 +++++++++- 3 files changed, 210 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ae1cc8d4..f007a0179 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -521,6 +521,38 @@ dependencies = [ "bytes 1.0.1", ] +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.10+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17fa3d1ac1ca21c5c4e36a97f3c3eb25084576f6fc47bf0139c1123434216c6c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cargo_toml" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d94e66797729c3a52b74980ec7992a9399aace3044bbcde9ce6bb98926abb673" +dependencies = [ + "serde", + "serde_derive", + "toml", +] + [[package]] name = "cc" version = "1.0.67" @@ -1236,6 +1268,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hostname" version = "0.3.1" @@ -1268,6 +1306,17 @@ dependencies = [ "http", ] +[[package]] +name = "http-body" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737" +dependencies = [ + "bytes 1.0.1", + "http", + "pin-project-lite 0.2.6", +] + [[package]] name = "httparse" version = "1.4.0" @@ -1313,7 +1362,7 @@ dependencies = [ "futures-util", "h2 0.2.7", "http", - "http-body", + "http-body 0.3.1", "httparse", "httpdate", "itoa", @@ -1325,6 +1374,30 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" +dependencies = [ + "bytes 1.0.1", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.2", + "http", + "http-body 0.4.1", + "httparse", + "httpdate", + "itoa", + "pin-project", + "socket2 0.4.0", + "tokio 1.5.0", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper-rustls" version = "0.21.0" @@ -1333,7 +1406,7 @@ checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" dependencies = [ "bytes 0.5.6", "futures-util", - "hyper", + "hyper 0.13.10", "log", "rustls 0.18.1", "tokio 0.2.25", @@ -1341,6 +1414,21 @@ dependencies = [ "webpki", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "futures-util", + "hyper 0.14.5", + "log", + "rustls 0.19.1", + "tokio 1.5.0", + "tokio-rustls 0.22.0", + "webpki", +] + [[package]] name = "idna" version = "0.2.3" @@ -1638,6 +1726,7 @@ dependencies = [ "async-trait", "byte-unit", "bytes 0.6.0", + "cargo_toml", "chrono", "crossbeam-channel", "dashmap", @@ -1649,6 +1738,7 @@ dependencies = [ "futures-util", "grenad", "heed", + "hex", "http", "indexmap", "itertools 0.10.0", @@ -1669,11 +1759,13 @@ dependencies = [ "rand 0.7.3", "rayon", "regex", + "reqwest 0.11.3", "rustls 0.19.1", "sentry", "serde", "serde_json", "serde_url_params", + "sha-1 0.9.4", "sha2", "siphasher", "slice-group-by", @@ -1687,6 +1779,7 @@ dependencies = [ "uuid", "vergen", "walkdir", + "zip", ] [[package]] @@ -2503,9 +2596,9 @@ dependencies = [ "futures-core", "futures-util", "http", - "http-body", - "hyper", - "hyper-rustls", + "http-body 0.3.1", + "hyper 0.13.10", + "hyper-rustls 0.21.0", "ipnet", "js-sys", "lazy_static", @@ -2528,6 +2621,41 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body 0.4.1", + "hyper 0.14.5", + "hyper-rustls 0.22.1", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project-lite 0.2.6", + "rustls 0.19.1", + "serde", + "serde_urlencoded", + "tokio 1.5.0", + "tokio-rustls 0.22.0", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.21.1", + "winreg", +] + [[package]] name = "retain_mut" version = "0.1.2" @@ -2691,7 +2819,7 @@ dependencies = [ "log", "rand 0.7.3", "regex", - "reqwest", + "reqwest 0.10.10", "rustc_version 0.2.3", "sentry-types", "uname", @@ -3275,6 +3403,15 @@ dependencies = [ "tokio 1.5.0", ] +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + [[package]] name = "tower-service" version = "0.3.1" @@ -3700,6 +3837,20 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zip" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c83dc9b784d252127720168abd71ea82bf8c3d96b17dc565b5e2a02854f2b27" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "flate2", + "thiserror", + "time 0.1.44", +] + [[package]] name = "zstd" version = "0.5.4+zstd.1.4.7" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e45311fa0..019a1a3f8 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -10,7 +10,13 @@ name = "meilisearch" path = "src/main.rs" [build-dependencies] +anyhow = "*" +cargo_toml = "0.9.0" +hex = "0.4.3" +reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false} +sha-1 = "0.9.4" vergen = "3.1.0" +zip = "0.5.12" [dependencies] actix-cors = "0.6.0-beta.1" @@ -96,3 +102,7 @@ default = ["sentry"] [target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" + +[package.metadata.mini-dashboard] +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.0/build.zip" +sha1 = "abb7bd8765b9fab38675958bc9d06088589c712c" diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index be4f4a1b6..542ae20e3 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -1,6 +1,14 @@ use std::env; +use std::fs::create_dir_all; +use std::io::Cursor; +use std::path::PathBuf; + +use anyhow::Context; +use sha1::{Sha1, Digest}; +use reqwest::blocking::get; use vergen::{generate_cargo_keys, ConstantsFlags}; +use cargo_toml::Manifest; fn main() { // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag @@ -11,6 +19,40 @@ fn main() { generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!"); if let Ok(_) = env::var("CARGO_FEATURE_MINI_DASHBOARD") { - todo!() + setup_mini_dashboard().expect("Could not load mini-dashboard assets"); } } + +fn setup_mini_dashboard() -> anyhow::Result<()> { + let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let cargo_toml = cargo_manifest_dir.join("Cargo.toml"); + + let manifest = Manifest::from_path(cargo_toml).unwrap(); + + let meta = &manifest + .package + .as_ref() + .context("package not specified in Cargo.toml")? + .metadata + .as_ref() + .context("no metadata specified in Cargo.toml")? + ["mini-dashboard"]; + + let url = meta["assets-url"].as_str().unwrap(); + + let dashboard_assets_bytes = get(url)? + .bytes()?; + + let mut hasher = Sha1::new(); + hasher.update(&dashboard_assets_bytes); + let sha1_dashboard = hex::encode(hasher.finalize()); + + assert_eq!(meta["sha1"].as_str().unwrap(), sha1_dashboard); + + let dashboard_dir = cargo_manifest_dir.join("mini-dashboard"); + create_dir_all(&dashboard_dir)?; + let cursor = Cursor::new(&dashboard_assets_bytes); + let mut zip = zip::read::ZipArchive::new(cursor)?; + zip.extract(&dashboard_dir)?; + Ok(()) +} From 6bcf20c70ec0af423603037723f0d2665e7ea963 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 21 Apr 2021 13:49:21 +0200 Subject: [PATCH 275/522] serve static site --- Cargo.lock | 90 ++++++++++++++++++++++++++--------- meilisearch-error/Cargo.toml | 2 +- meilisearch-http/Cargo.toml | 16 ++++--- meilisearch-http/build.rs | 2 + meilisearch-http/src/error.rs | 10 ++-- meilisearch-http/src/lib.rs | 29 +++++++++-- meilisearch-http/src/main.rs | 1 + 7 files changed, 114 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f007a0179..8065d7ae4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,8 +19,7 @@ dependencies = [ [[package]] name = "actix-cors" version = "0.6.0-beta.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa50c395d15e5946cf04bccb1edef583e9fd42aa6710a1f89a725c5e2c4c5503" +source = "git+https://github.com/MarinPostma/actix-extras.git?rev=2dac1a4#2dac1a421619bf7b386dea63d3ae25a3bc4abc43" dependencies = [ "actix-service", "actix-web", @@ -33,9 +32,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.0.0-beta.5" +version = "3.0.0-beta.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb9c5d7ceb490d6565156ae1d4d467db17da1759425c65a2e36ac5e182e014e2" +checksum = "59d51c2ba06062e698a5d212d860e9fb2afc931c285ede687aaae896c8150347" dependencies = [ "actix-codec", "actix-rt", @@ -48,7 +47,6 @@ dependencies = [ "brotli2", "bytes 1.0.1", "bytestring", - "cookie", "derive_more", "encoding_rs", "flate2", @@ -63,14 +61,13 @@ dependencies = [ "log", "mime", "once_cell", + "paste", "percent-encoding", "pin-project", "pin-project-lite 0.2.6", "rand 0.8.3", "regex", "serde", - "serde_json", - "serde_urlencoded", "sha-1 0.9.4", "smallvec", "time 0.2.26", @@ -113,9 +110,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.0.0-beta.4" +version = "2.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0872f02a1871257ef09c5a269dce5dc5fea5ccf502adbf5d39f118913b61411c" +checksum = "26369215fcc3b0176018b3b68756a8bcc275bb000e6212e454944913a1f9bf87" dependencies = [ "actix-rt", "actix-service", @@ -130,11 +127,12 @@ dependencies = [ [[package]] name = "actix-service" -version = "2.0.0-beta.5" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf82340ad9f4e4caf43737fd3bbc999778a268015cdc54675f60af6240bd2b05" +checksum = "77f5f9d66a8730d0fae62c26f3424f5751e5518086628a40b7ab6fca4a705034" dependencies = [ "futures-core", + "paste", "pin-project-lite 0.2.6", ] @@ -169,9 +167,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.0.0-beta.5" +version = "4.0.0-beta.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6de19cc341c2e68b1ee126de171e86b610b5bbcecc660d1250ebed73e0257bd6" +checksum = "ff12e933051557d700b0fcad20fe25b9ca38395cc87bbc5aeaddaef17b937a2f" dependencies = [ "actix-codec", "actix-http", @@ -185,11 +183,13 @@ dependencies = [ "actix-web-codegen", "ahash 0.7.2", "bytes 1.0.1", + "cookie", "derive_more", "either", "encoding_rs", "futures-core", "futures-util", + "itoa", "language-tags", "log", "mime", @@ -216,6 +216,20 @@ dependencies = [ "syn 1.0.70", ] +[[package]] +name = "actix-web-static-files" +version = "3.0.5" +source = "git+https://github.com/MarinPostma/actix-web-static-files.git?branch=actix-web-4#6db8c3e2940d61659581492b5e9c9b9062567613" +dependencies = [ + "actix-service", + "actix-web", + "change-detection", + "derive_more", + "futures", + "mime_guess", + "path-slash", +] + [[package]] name = "addr2line" version = "0.14.1" @@ -583,6 +597,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "change-detection" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "159fa412eae48a1d94d0b9ecdb85c97ce56eb2a347c62394d3fdbf221adabc1a" +dependencies = [ + "path-matchers", + "path-slash", +] + [[package]] name = "character_converter" version = "1.0.0" @@ -635,9 +659,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "cookie" -version = "0.14.4" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" +checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627" dependencies = [ "percent-encoding", "time 0.2.26", @@ -1329,6 +1353,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" +[[package]] +name = "httpdate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" + [[package]] name = "human_format" version = "1.0.3" @@ -1364,7 +1394,7 @@ dependencies = [ "http", "http-body 0.3.1", "httparse", - "httpdate", + "httpdate 0.3.2", "itoa", "pin-project", "socket2 0.3.19", @@ -1376,9 +1406,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" +checksum = "5f006b8784cfb01fe7aa9c46f5f5cd4cf5c85a8c612a0653ec97642979062665" dependencies = [ "bytes 1.0.1", "futures-channel", @@ -1388,7 +1418,7 @@ dependencies = [ "http", "http-body 0.4.1", "httparse", - "httpdate", + "httpdate 1.0.0", "itoa", "pin-project", "socket2 0.4.0", @@ -1421,7 +1451,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" dependencies = [ "futures-util", - "hyper 0.14.5", + "hyper 0.14.6", "log", "rustls 0.19.1", "tokio 1.5.0", @@ -1719,6 +1749,7 @@ dependencies = [ "actix-rt", "actix-service", "actix-web", + "actix-web-static-files", "anyhow", "assert-json-diff", "async-compression", @@ -2126,6 +2157,21 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" +[[package]] +name = "path-matchers" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36cd9b72a47679ec193a5f0229d9ab686b7bd45e1fbc59ccf953c9f3d83f7b2b" +dependencies = [ + "glob", +] + +[[package]] +name = "path-slash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cacbb3c4ff353b534a67fb8d7524d00229da4cb1dc8c79f4db96e375ab5b619" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -2634,7 +2680,7 @@ dependencies = [ "futures-util", "http", "http-body 0.4.1", - "hyper 0.14.5", + "hyper 0.14.6", "hyper-rustls 0.22.1", "ipnet", "js-sys", @@ -2812,7 +2858,7 @@ dependencies = [ "env_logger 0.7.1", "failure", "hostname", - "httpdate", + "httpdate 0.3.2", "im", "lazy_static", "libc", diff --git a/meilisearch-error/Cargo.toml b/meilisearch-error/Cargo.toml index 7673e1391..1340b0020 100644 --- a/meilisearch-error/Cargo.toml +++ b/meilisearch-error/Cargo.toml @@ -5,4 +5,4 @@ authors = ["marin "] edition = "2018" [dependencies] -actix-http = "=3.0.0-beta.5" +actix-http = "=3.0.0-beta.6" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 019a1a3f8..e2edbccca 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -5,11 +5,14 @@ edition = "2018" license = "MIT" name = "meilisearch-http" version = "0.21.0-alpha.3" +build = "build.rs" + [[bin]] name = "meilisearch" path = "src/main.rs" [build-dependencies] +actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", branch = "actix-web-4" } anyhow = "*" cargo_toml = "0.9.0" hex = "0.4.3" @@ -19,10 +22,11 @@ vergen = "3.1.0" zip = "0.5.12" [dependencies] -actix-cors = "0.6.0-beta.1" -actix-http = { version = "=3.0.0-beta.5" } -actix-service = "=2.0.0-beta.5" -actix-web = { version = "=4.0.0-beta.5", features = ["rustls"] } +actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "2dac1a4"} +actix-http = { version = "=3.0.0-beta.6" } +actix-service = "2.0.0" +actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] } +actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", branch = "actix-web-4", optional = true } anyhow = "1.0.36" async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } async-stream = "0.3.0" @@ -51,6 +55,7 @@ memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.1" } mime = "0.3.16" once_cell = "1.5.2" +oxidized-json-checker = "0.3.2" parking_lot = "0.11.1" rand = "0.7.3" rayon = "1.5.0" @@ -67,7 +72,6 @@ tempfile = "3.1.0" thiserror = "1.0.24" tokio = { version = "1", features = ["full"] } uuid = "0.8.2" -oxidized-json-checker = "0.3.2" walkdir = "2.3.2" obkv = "0.1.1" @@ -97,7 +101,7 @@ tempdir = "0.3.7" urlencoding = "1.1.1" [features] -mini-dashboard = ["default"] +mini-dashboard = ["default", "actix-web-static-files"] default = ["sentry"] [target.'cfg(target_os = "linux")'.dependencies] diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index 542ae20e3..ef3b67271 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use anyhow::Context; use sha1::{Sha1, Digest}; use reqwest::blocking::get; +use actix_web_static_files::resource_dir; use vergen::{generate_cargo_keys, ConstantsFlags}; use cargo_toml::Manifest; @@ -54,5 +55,6 @@ fn setup_mini_dashboard() -> anyhow::Result<()> { let cursor = Cursor::new(&dashboard_assets_bytes); let mut zip = zip::read::ZipArchive::new(cursor)?; zip.extract(&dashboard_dir)?; + resource_dir(&dashboard_dir).build()?; Ok(()) } diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 8bfdb3573..6489716ca 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -2,7 +2,8 @@ use std::error; use std::fmt; use actix_web as aweb; -use actix_web::dev::HttpResponseBuilder; +use actix_web::body::Body; +use actix_web::dev::BaseHttpResponseBuilder; use actix_web::error::{JsonPayloadError, QueryPayloadError}; use actix_web::http::Error as HttpError; use actix_web::http::StatusCode; @@ -71,8 +72,9 @@ impl Serialize for ResponseError { } impl aweb::error::ResponseError for ResponseError { - fn error_response(&self) -> aweb::HttpResponse { - HttpResponseBuilder::new(self.status_code()).json(&self) + fn error_response(&self) -> aweb::BaseHttpResponse { + let json = serde_json::to_vec(self).unwrap(); + BaseHttpResponseBuilder::new(self.status_code()).body(json) } fn status_code(&self) -> StatusCode { @@ -297,6 +299,7 @@ impl From for Error { JsonPayloadError::Payload(err) => { Error::BadRequest(format!("Problem while decoding the request: {}", err)) } + e => Error::Internal(format!("Unexpected Json error: {}", e)) } } } @@ -307,6 +310,7 @@ impl From for Error { QueryPayloadError::Deserialize(err) => { Error::BadRequest(format!("Invalid query parameters: {}", err)) } + e => Error::Internal(format!("Unexpected query payload error: {}", e)) } } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 7665b5694..fcbd6e6cb 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -9,6 +9,7 @@ pub mod routes; pub use self::data::Data; pub use option::Opt; + #[macro_export] macro_rules! create_app { ($data:expr, $enable_frontend:expr) => {{ @@ -19,6 +20,14 @@ macro_rules! create_app { use meilisearch_http::error::payload_error_handler; use meilisearch_http::routes::*; + #[cfg(feature = "mini-dashboard")] + use actix_web_static_files::ResourceFiles; + + #[cfg(feature = "mini-dashboard")] + mod dashboard { + include!(concat!(env!("OUT_DIR"), "/generated.rs")); + } + let app = App::new() .data($data.clone()) .app_data( @@ -40,16 +49,28 @@ macro_rules! create_app { .configure(stats::services) .configure(key::services); //.configure(routes::dump::services); + #[cfg(feature = "mini-dashboard")] let app = if $enable_frontend { - app.service(load_html).service(load_css) + let generated = dashboard::generate(); + let keys = generated.keys().collect::>(); + println!("served files {:?}", keys); + let service = ResourceFiles::new("/", generated); + app.service(service) } else { app.service(running) }; + + #[cfg(not(feature = "mini-dashboard"))] + let app = app.service(running); + + println!("here\n\n\n"); app.wrap( Cors::default() - .send_wildcard() - .allowed_headers(vec!["content-type", "x-meili-api-key"]) - .max_age(86_400), // 24h + .send_wildcard() + .allowed_headers(vec!["content-type", "x-meili-api-key"]) + .allow_any_origin() + .allow_any_method() + .max_age(86_400) // 24h ) .wrap(middleware::Logger::default()) .wrap(middleware::Compress::default()) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index eab7f4ac6..1163c4f24 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -79,6 +79,7 @@ async fn main() -> Result<(), MainError> { Ok(()) } +#[allow(unused_variables)] async fn run_http( data: Data, opt: Opt, From 6af769af20039d258126f2983b36d0145a2cb38d Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 21 Apr 2021 14:40:22 +0200 Subject: [PATCH 276/522] bump mini-dashboard --- meilisearch-http/Cargo.toml | 4 ++-- meilisearch-http/src/lib.rs | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e2edbccca..369f93376 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -108,5 +108,5 @@ default = ["sentry"] jemallocator = "0.3.2" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.0/build.zip" -sha1 = "abb7bd8765b9fab38675958bc9d06088589c712c" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.1/build.zip" +sha1 = "f4247f8f534214e2811637e0555347c3f6bf5794" diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index fcbd6e6cb..44166d1ff 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -53,7 +53,6 @@ macro_rules! create_app { let app = if $enable_frontend { let generated = dashboard::generate(); let keys = generated.keys().collect::>(); - println!("served files {:?}", keys); let service = ResourceFiles::new("/", generated); app.service(service) } else { @@ -63,7 +62,6 @@ macro_rules! create_app { #[cfg(not(feature = "mini-dashboard"))] let app = app.service(running); - println!("here\n\n\n"); app.wrap( Cors::default() .send_wildcard() From 90f57c13290d0ad858dd7365079e41a4256859af Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 22 Apr 2021 11:22:09 +0200 Subject: [PATCH 277/522] update CI & Dockerfile --- .github/workflows/create_artifacts.yml | 2 +- Dockerfile | 4 ++-- meilisearch-http/Cargo.toml | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_artifacts.yml b/.github/workflows/create_artifacts.yml index 94378ba83..06b116c28 100644 --- a/.github/workflows/create_artifacts.yml +++ b/.github/workflows/create_artifacts.yml @@ -28,7 +28,7 @@ jobs: rust-version: stable - uses: actions/checkout@v1 - name: Build - run: cargo build --release --locked + run: cargo build --release --locked --features mini-dashboard - name: Upload binaries to release uses: svenstaro/upload-release-action@v1-release with: diff --git a/Dockerfile b/Dockerfile index 8c2648512..4f52c61d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" # Create dummy main.rs files for each workspace member to be able to compile all the dependencies RUN find . -type d -name "meilisearch-*" | xargs -I{} sh -c 'mkdir {}/src; echo "fn main() { }" > {}/src/main.rs;' # Use `cargo build` instead of `cargo vendor` because we need to not only download but compile dependencies too -RUN $HOME/.cargo/bin/cargo build --release +RUN $HOME/.cargo/bin/cargo build --release --features mini-dashboard # Cleanup dummy main.rs files RUN find . -path "*/src/main.rs" -delete @@ -29,7 +29,7 @@ ARG COMMIT_DATE ENV COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE} COPY . . -RUN $HOME/.cargo/bin/cargo build --release +RUN $HOME/.cargo/bin/cargo build --release --features mini-dashboard # Run FROM alpine:3.10 diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 369f93376..741c2eebe 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -5,7 +5,6 @@ edition = "2018" license = "MIT" name = "meilisearch-http" version = "0.21.0-alpha.3" -build = "build.rs" [[bin]] name = "meilisearch" From e4bd1bc5ceea9360cad9aadc0e62c0d4bf325841 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 22 Apr 2021 11:42:41 +0200 Subject: [PATCH 278/522] update actix-web-static-file rev --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8065d7ae4..879e43de9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "actix-web-static-files" version = "3.0.5" -source = "git+https://github.com/MarinPostma/actix-web-static-files.git?branch=actix-web-4#6db8c3e2940d61659581492b5e9c9b9062567613" +source = "git+https://github.com/MarinPostma/actix-web-static-files.git?rev=6db8c3e#6db8c3e2940d61659581492b5e9c9b9062567613" dependencies = [ "actix-service", "actix-web", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 741c2eebe..743c59de2 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -11,12 +11,13 @@ name = "meilisearch" path = "src/main.rs" [build-dependencies] -actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", branch = "actix-web-4" } +actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e" } anyhow = "*" cargo_toml = "0.9.0" hex = "0.4.3" reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false} sha-1 = "0.9.4" +tempfile = "3.1.0" vergen = "3.1.0" zip = "0.5.12" @@ -25,7 +26,7 @@ actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "2 actix-http = { version = "=3.0.0-beta.6" } actix-service = "2.0.0" actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] } -actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", branch = "actix-web-4", optional = true } +actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true } anyhow = "1.0.36" async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } async-stream = "0.3.0" From c2461e50664acc47fcf1180e73dd8163b0b0f262 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 22 Apr 2021 12:39:23 +0200 Subject: [PATCH 279/522] review fixes --- .github/workflows/create_artifacts.yml | 2 +- Dockerfile | 4 +- meilisearch-http/Cargo.toml | 32 ++- meilisearch-http/build.rs | 120 +++++---- meilisearch-http/public/bulma.min.css | 1 - meilisearch-http/public/interface.html | 333 ------------------------- meilisearch-http/src/lib.rs | 2 - meilisearch-http/src/main.rs | 18 +- meilisearch-http/src/routes/mod.rs | 15 -- 9 files changed, 102 insertions(+), 425 deletions(-) delete mode 100644 meilisearch-http/public/bulma.min.css delete mode 100644 meilisearch-http/public/interface.html diff --git a/.github/workflows/create_artifacts.yml b/.github/workflows/create_artifacts.yml index 06b116c28..94378ba83 100644 --- a/.github/workflows/create_artifacts.yml +++ b/.github/workflows/create_artifacts.yml @@ -28,7 +28,7 @@ jobs: rust-version: stable - uses: actions/checkout@v1 - name: Build - run: cargo build --release --locked --features mini-dashboard + run: cargo build --release --locked - name: Upload binaries to release uses: svenstaro/upload-release-action@v1-release with: diff --git a/Dockerfile b/Dockerfile index 4f52c61d0..8c2648512 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" # Create dummy main.rs files for each workspace member to be able to compile all the dependencies RUN find . -type d -name "meilisearch-*" | xargs -I{} sh -c 'mkdir {}/src; echo "fn main() { }" > {}/src/main.rs;' # Use `cargo build` instead of `cargo vendor` because we need to not only download but compile dependencies too -RUN $HOME/.cargo/bin/cargo build --release --features mini-dashboard +RUN $HOME/.cargo/bin/cargo build --release # Cleanup dummy main.rs files RUN find . -path "*/src/main.rs" -delete @@ -29,7 +29,7 @@ ARG COMMIT_DATE ENV COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE} COPY . . -RUN $HOME/.cargo/bin/cargo build --release --features mini-dashboard +RUN $HOME/.cargo/bin/cargo build --release # Run FROM alpine:3.10 diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 743c59de2..a5d5bc14d 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -11,15 +11,15 @@ name = "meilisearch" path = "src/main.rs" [build-dependencies] -actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e" } -anyhow = "*" -cargo_toml = "0.9.0" -hex = "0.4.3" -reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false} -sha-1 = "0.9.4" -tempfile = "3.1.0" +actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true } +anyhow = { version = "*", optional = true } +cargo_toml = { version = "0.9.0", optional = true } +hex = { version = "0.4.3", optional = true } +reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true } +sha-1 = { version = "0.9.4", optional = true } +tempfile = { version = "3.1.0", optional = true } vergen = "3.1.0" -zip = "0.5.12" +zip = { version = "0.5.12", optional = true } [dependencies] actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "2dac1a4"} @@ -71,7 +71,7 @@ tar = "0.4.29" tempfile = "3.1.0" thiserror = "1.0.24" tokio = { version = "1", features = ["full"] } -uuid = "0.8.2" +uuid = { version = "0.8.2", features = ["serde"] } walkdir = "2.3.2" obkv = "0.1.1" @@ -101,8 +101,18 @@ tempdir = "0.3.7" urlencoding = "1.1.1" [features] -mini-dashboard = ["default", "actix-web-static-files"] -default = ["sentry"] +mini-dashboard = [ + "actix-web-static-files", + "anyhow", + "cargo_toml", + "hex", + "reqwest", + "sha-1", + "tempfile", + "zip", +] +telemetry = ["sentry"] +default = ["telemetry", "mini-dashboard"] [target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index ef3b67271..8cc743543 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -1,15 +1,4 @@ -use std::env; -use std::fs::create_dir_all; -use std::io::Cursor; -use std::path::PathBuf; - -use anyhow::Context; -use sha1::{Sha1, Digest}; -use reqwest::blocking::get; -use actix_web_static_files::resource_dir; - use vergen::{generate_cargo_keys, ConstantsFlags}; -use cargo_toml::Manifest; fn main() { // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag @@ -19,42 +8,77 @@ fn main() { // Generate the 'cargo:' key output generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!"); - if let Ok(_) = env::var("CARGO_FEATURE_MINI_DASHBOARD") { - setup_mini_dashboard().expect("Could not load mini-dashboard assets"); + #[cfg(feature = "mini-dashboard")] + mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets"); +} + +#[cfg(feature = "mini-dashboard")] +mod mini_dashboard { + use std::env; + use std::fs::{create_dir_all, File, OpenOptions}; + use std::io::{Cursor, Read, Write}; + use std::path::PathBuf; + + use actix_web_static_files::resource_dir; + use anyhow::Context; + use cargo_toml::Manifest; + use reqwest::blocking::get; + use sha1::{Digest, Sha1}; + + pub fn setup_mini_dashboard() -> anyhow::Result<()> { + let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let cargo_toml = cargo_manifest_dir.join("Cargo.toml"); + + let sha1_path = cargo_manifest_dir.join(".mini-dashboard.sha1"); + let dashboard_dir = cargo_manifest_dir.join("mini-dashboard"); + + let manifest = Manifest::from_path(cargo_toml).unwrap(); + + let meta = &manifest + .package + .as_ref() + .context("package not specified in Cargo.toml")? + .metadata + .as_ref() + .context("no metadata specified in Cargo.toml")?["mini-dashboard"]; + + // Check if there already is a dashboard built, and if it is up to date. + if sha1_path.exists() && dashboard_dir.exists() { + let mut sha1_file = File::open(&sha1_path)?; + let mut sha1 = String::new(); + sha1_file.read_to_string(&mut sha1)?; + if sha1 == meta["sha1"].as_str().unwrap() { + // Nothing to do. + return Ok(()) + } + } + + let url = meta["assets-url"].as_str().unwrap(); + + let dashboard_assets_bytes = get(url)?.bytes()?; + + let mut hasher = Sha1::new(); + hasher.update(&dashboard_assets_bytes); + let sha1 = hex::encode(hasher.finalize()); + + assert_eq!(meta["sha1"].as_str().unwrap(), sha1, "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml"); + + create_dir_all(&dashboard_dir)?; + let cursor = Cursor::new(&dashboard_assets_bytes); + let mut zip = zip::read::ZipArchive::new(cursor)?; + zip.extract(&dashboard_dir)?; + resource_dir(&dashboard_dir).build()?; + + // Write the sha1 for the dashboard back to file. + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(sha1_path)?; + + file.write_all(sha1.as_bytes())?; + file.flush()?; + + Ok(()) } } - -fn setup_mini_dashboard() -> anyhow::Result<()> { - let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let cargo_toml = cargo_manifest_dir.join("Cargo.toml"); - - let manifest = Manifest::from_path(cargo_toml).unwrap(); - - let meta = &manifest - .package - .as_ref() - .context("package not specified in Cargo.toml")? - .metadata - .as_ref() - .context("no metadata specified in Cargo.toml")? - ["mini-dashboard"]; - - let url = meta["assets-url"].as_str().unwrap(); - - let dashboard_assets_bytes = get(url)? - .bytes()?; - - let mut hasher = Sha1::new(); - hasher.update(&dashboard_assets_bytes); - let sha1_dashboard = hex::encode(hasher.finalize()); - - assert_eq!(meta["sha1"].as_str().unwrap(), sha1_dashboard); - - let dashboard_dir = cargo_manifest_dir.join("mini-dashboard"); - create_dir_all(&dashboard_dir)?; - let cursor = Cursor::new(&dashboard_assets_bytes); - let mut zip = zip::read::ZipArchive::new(cursor)?; - zip.extract(&dashboard_dir)?; - resource_dir(&dashboard_dir).build()?; - Ok(()) -} diff --git a/meilisearch-http/public/bulma.min.css b/meilisearch-http/public/bulma.min.css deleted file mode 100644 index d0570ff03..000000000 --- a/meilisearch-http/public/bulma.min.css +++ /dev/null @@ -1 +0,0 @@ -/*! bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */@-webkit-keyframes spinAround{from{transform:rotate(0)}to{transform:rotate(359deg)}}@keyframes spinAround{from{transform:rotate(0)}to{transform:rotate(359deg)}}.breadcrumb,.button,.delete,.file,.is-unselectable,.modal-close,.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous,.tabs{-webkit-touch-callout:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.navbar-link:not(.is-arrowless)::after,.select:not(.is-multiple):not(.is-loading)::after{border:3px solid transparent;border-radius:2px;border-right:0;border-top:0;content:" ";display:block;height:.625em;margin-top:-.4375em;pointer-events:none;position:absolute;top:50%;transform:rotate(-45deg);transform-origin:center;width:.625em}.block:not(:last-child),.box:not(:last-child),.breadcrumb:not(:last-child),.content:not(:last-child),.highlight:not(:last-child),.level:not(:last-child),.list:not(:last-child),.message:not(:last-child),.notification:not(:last-child),.pagination:not(:last-child),.progress:not(:last-child),.subtitle:not(:last-child),.table-container:not(:last-child),.table:not(:last-child),.tabs:not(:last-child),.title:not(:last-child){margin-bottom:1.5rem}.delete,.modal-close{-moz-appearance:none;-webkit-appearance:none;background-color:rgba(10,10,10,.2);border:none;border-radius:290486px;cursor:pointer;pointer-events:auto;display:inline-block;flex-grow:0;flex-shrink:0;font-size:0;height:20px;max-height:20px;max-width:20px;min-height:20px;min-width:20px;outline:0;position:relative;vertical-align:top;width:20px}.delete::after,.delete::before,.modal-close::after,.modal-close::before{background-color:#fff;content:"";display:block;left:50%;position:absolute;top:50%;transform:translateX(-50%) translateY(-50%) rotate(45deg);transform-origin:center center}.delete::before,.modal-close::before{height:2px;width:50%}.delete::after,.modal-close::after{height:50%;width:2px}.delete:focus,.delete:hover,.modal-close:focus,.modal-close:hover{background-color:rgba(10,10,10,.3)}.delete:active,.modal-close:active{background-color:rgba(10,10,10,.4)}.is-small.delete,.is-small.modal-close{height:16px;max-height:16px;max-width:16px;min-height:16px;min-width:16px;width:16px}.is-medium.delete,.is-medium.modal-close{height:24px;max-height:24px;max-width:24px;min-height:24px;min-width:24px;width:24px}.is-large.delete,.is-large.modal-close{height:32px;max-height:32px;max-width:32px;min-height:32px;min-width:32px;width:32px}.button.is-loading::after,.control.is-loading::after,.loader,.select.is-loading::after{-webkit-animation:spinAround .5s infinite linear;animation:spinAround .5s infinite linear;border:2px solid #dbdbdb;border-radius:290486px;border-right-color:transparent;border-top-color:transparent;content:"";display:block;height:1em;position:relative;width:1em}.hero-video,.image.is-16by9 .has-ratio,.image.is-16by9 img,.image.is-1by1 .has-ratio,.image.is-1by1 img,.image.is-1by2 .has-ratio,.image.is-1by2 img,.image.is-1by3 .has-ratio,.image.is-1by3 img,.image.is-2by1 .has-ratio,.image.is-2by1 img,.image.is-2by3 .has-ratio,.image.is-2by3 img,.image.is-3by1 .has-ratio,.image.is-3by1 img,.image.is-3by2 .has-ratio,.image.is-3by2 img,.image.is-3by4 .has-ratio,.image.is-3by4 img,.image.is-3by5 .has-ratio,.image.is-3by5 img,.image.is-4by3 .has-ratio,.image.is-4by3 img,.image.is-4by5 .has-ratio,.image.is-4by5 img,.image.is-5by3 .has-ratio,.image.is-5by3 img,.image.is-5by4 .has-ratio,.image.is-5by4 img,.image.is-9by16 .has-ratio,.image.is-9by16 img,.image.is-square .has-ratio,.image.is-square img,.is-overlay,.modal,.modal-background{bottom:0;left:0;position:absolute;right:0;top:0}.button,.file-cta,.file-name,.input,.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous,.select select,.textarea{-moz-appearance:none;-webkit-appearance:none;align-items:center;border:1px solid transparent;border-radius:4px;box-shadow:none;display:inline-flex;font-size:1rem;height:2.5em;justify-content:flex-start;line-height:1.5;padding-bottom:calc(.5em - 1px);padding-left:calc(.75em - 1px);padding-right:calc(.75em - 1px);padding-top:calc(.5em - 1px);position:relative;vertical-align:top}.button:active,.button:focus,.file-cta:active,.file-cta:focus,.file-name:active,.file-name:focus,.input:active,.input:focus,.is-active.button,.is-active.file-cta,.is-active.file-name,.is-active.input,.is-active.pagination-ellipsis,.is-active.pagination-link,.is-active.pagination-next,.is-active.pagination-previous,.is-active.textarea,.is-focused.button,.is-focused.file-cta,.is-focused.file-name,.is-focused.input,.is-focused.pagination-ellipsis,.is-focused.pagination-link,.is-focused.pagination-next,.is-focused.pagination-previous,.is-focused.textarea,.pagination-ellipsis:active,.pagination-ellipsis:focus,.pagination-link:active,.pagination-link:focus,.pagination-next:active,.pagination-next:focus,.pagination-previous:active,.pagination-previous:focus,.select select.is-active,.select select.is-focused,.select select:active,.select select:focus,.textarea:active,.textarea:focus{outline:0}.button[disabled],.file-cta[disabled],.file-name[disabled],.input[disabled],.pagination-ellipsis[disabled],.pagination-link[disabled],.pagination-next[disabled],.pagination-previous[disabled],.select fieldset[disabled] select,.select select[disabled],.textarea[disabled],fieldset[disabled] .button,fieldset[disabled] .file-cta,fieldset[disabled] .file-name,fieldset[disabled] .input,fieldset[disabled] .pagination-ellipsis,fieldset[disabled] .pagination-link,fieldset[disabled] .pagination-next,fieldset[disabled] .pagination-previous,fieldset[disabled] .select select,fieldset[disabled] .textarea{cursor:not-allowed}/*! minireset.css v0.0.6 | MIT License | github.com/jgthms/minireset.css */blockquote,body,dd,dl,dt,fieldset,figure,h1,h2,h3,h4,h5,h6,hr,html,iframe,legend,li,ol,p,pre,textarea,ul{margin:0;padding:0}h1,h2,h3,h4,h5,h6{font-size:100%;font-weight:400}ul{list-style:none}button,input,select,textarea{margin:0}html{box-sizing:border-box}*,::after,::before{box-sizing:inherit}img,video{height:auto;max-width:100%}iframe{border:0}table{border-collapse:collapse;border-spacing:0}td,th{padding:0}td:not([align]),th:not([align]){text-align:left}html{background-color:#fff;font-size:16px;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;min-width:300px;overflow-x:hidden;overflow-y:scroll;text-rendering:optimizeLegibility;-webkit-text-size-adjust:100%;-moz-text-size-adjust:100%;-ms-text-size-adjust:100%;text-size-adjust:100%}article,aside,figure,footer,header,hgroup,section{display:block}body,button,input,select,textarea{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif}code,pre{-moz-osx-font-smoothing:auto;-webkit-font-smoothing:auto;font-family:monospace}body{color:#4a4a4a;font-size:1em;font-weight:400;line-height:1.5}a{color:#3273dc;cursor:pointer;text-decoration:none}a strong{color:currentColor}a:hover{color:#363636}code{background-color:#f5f5f5;color:#f14668;font-size:.875em;font-weight:400;padding:.25em .5em .25em}hr{background-color:#f5f5f5;border:none;display:block;height:2px;margin:1.5rem 0}img{height:auto;max-width:100%}input[type=checkbox],input[type=radio]{vertical-align:baseline}small{font-size:.875em}span{font-style:inherit;font-weight:inherit}strong{color:#363636;font-weight:700}fieldset{border:none}pre{-webkit-overflow-scrolling:touch;background-color:#f5f5f5;color:#4a4a4a;font-size:.875em;overflow-x:auto;padding:1.25rem 1.5rem;white-space:pre;word-wrap:normal}pre code{background-color:transparent;color:currentColor;font-size:1em;padding:0}table td,table th{vertical-align:top}table td:not([align]),table th:not([align]){text-align:left}table th{color:#363636}.is-clearfix::after{clear:both;content:" ";display:table}.is-pulled-left{float:left!important}.is-pulled-right{float:right!important}.is-clipped{overflow:hidden!important}.is-size-1{font-size:3rem!important}.is-size-2{font-size:2.5rem!important}.is-size-3{font-size:2rem!important}.is-size-4{font-size:1.5rem!important}.is-size-5{font-size:1.25rem!important}.is-size-6{font-size:1rem!important}.is-size-7{font-size:.75rem!important}@media screen and (max-width:768px){.is-size-1-mobile{font-size:3rem!important}.is-size-2-mobile{font-size:2.5rem!important}.is-size-3-mobile{font-size:2rem!important}.is-size-4-mobile{font-size:1.5rem!important}.is-size-5-mobile{font-size:1.25rem!important}.is-size-6-mobile{font-size:1rem!important}.is-size-7-mobile{font-size:.75rem!important}}@media screen and (min-width:769px),print{.is-size-1-tablet{font-size:3rem!important}.is-size-2-tablet{font-size:2.5rem!important}.is-size-3-tablet{font-size:2rem!important}.is-size-4-tablet{font-size:1.5rem!important}.is-size-5-tablet{font-size:1.25rem!important}.is-size-6-tablet{font-size:1rem!important}.is-size-7-tablet{font-size:.75rem!important}}@media screen and (max-width:1023px){.is-size-1-touch{font-size:3rem!important}.is-size-2-touch{font-size:2.5rem!important}.is-size-3-touch{font-size:2rem!important}.is-size-4-touch{font-size:1.5rem!important}.is-size-5-touch{font-size:1.25rem!important}.is-size-6-touch{font-size:1rem!important}.is-size-7-touch{font-size:.75rem!important}}@media screen and (min-width:1024px){.is-size-1-desktop{font-size:3rem!important}.is-size-2-desktop{font-size:2.5rem!important}.is-size-3-desktop{font-size:2rem!important}.is-size-4-desktop{font-size:1.5rem!important}.is-size-5-desktop{font-size:1.25rem!important}.is-size-6-desktop{font-size:1rem!important}.is-size-7-desktop{font-size:.75rem!important}}@media screen and (min-width:1216px){.is-size-1-widescreen{font-size:3rem!important}.is-size-2-widescreen{font-size:2.5rem!important}.is-size-3-widescreen{font-size:2rem!important}.is-size-4-widescreen{font-size:1.5rem!important}.is-size-5-widescreen{font-size:1.25rem!important}.is-size-6-widescreen{font-size:1rem!important}.is-size-7-widescreen{font-size:.75rem!important}}@media screen and (min-width:1408px){.is-size-1-fullhd{font-size:3rem!important}.is-size-2-fullhd{font-size:2.5rem!important}.is-size-3-fullhd{font-size:2rem!important}.is-size-4-fullhd{font-size:1.5rem!important}.is-size-5-fullhd{font-size:1.25rem!important}.is-size-6-fullhd{font-size:1rem!important}.is-size-7-fullhd{font-size:.75rem!important}}.has-text-centered{text-align:center!important}.has-text-justified{text-align:justify!important}.has-text-left{text-align:left!important}.has-text-right{text-align:right!important}@media screen and (max-width:768px){.has-text-centered-mobile{text-align:center!important}}@media screen and (min-width:769px),print{.has-text-centered-tablet{text-align:center!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-centered-tablet-only{text-align:center!important}}@media screen and (max-width:1023px){.has-text-centered-touch{text-align:center!important}}@media screen and (min-width:1024px){.has-text-centered-desktop{text-align:center!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-centered-desktop-only{text-align:center!important}}@media screen and (min-width:1216px){.has-text-centered-widescreen{text-align:center!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-centered-widescreen-only{text-align:center!important}}@media screen and (min-width:1408px){.has-text-centered-fullhd{text-align:center!important}}@media screen and (max-width:768px){.has-text-justified-mobile{text-align:justify!important}}@media screen and (min-width:769px),print{.has-text-justified-tablet{text-align:justify!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-justified-tablet-only{text-align:justify!important}}@media screen and (max-width:1023px){.has-text-justified-touch{text-align:justify!important}}@media screen and (min-width:1024px){.has-text-justified-desktop{text-align:justify!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-justified-desktop-only{text-align:justify!important}}@media screen and (min-width:1216px){.has-text-justified-widescreen{text-align:justify!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-justified-widescreen-only{text-align:justify!important}}@media screen and (min-width:1408px){.has-text-justified-fullhd{text-align:justify!important}}@media screen and (max-width:768px){.has-text-left-mobile{text-align:left!important}}@media screen and (min-width:769px),print{.has-text-left-tablet{text-align:left!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-left-tablet-only{text-align:left!important}}@media screen and (max-width:1023px){.has-text-left-touch{text-align:left!important}}@media screen and (min-width:1024px){.has-text-left-desktop{text-align:left!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-left-desktop-only{text-align:left!important}}@media screen and (min-width:1216px){.has-text-left-widescreen{text-align:left!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-left-widescreen-only{text-align:left!important}}@media screen and (min-width:1408px){.has-text-left-fullhd{text-align:left!important}}@media screen and (max-width:768px){.has-text-right-mobile{text-align:right!important}}@media screen and (min-width:769px),print{.has-text-right-tablet{text-align:right!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-right-tablet-only{text-align:right!important}}@media screen and (max-width:1023px){.has-text-right-touch{text-align:right!important}}@media screen and (min-width:1024px){.has-text-right-desktop{text-align:right!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-right-desktop-only{text-align:right!important}}@media screen and (min-width:1216px){.has-text-right-widescreen{text-align:right!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-right-widescreen-only{text-align:right!important}}@media screen and (min-width:1408px){.has-text-right-fullhd{text-align:right!important}}.is-capitalized{text-transform:capitalize!important}.is-lowercase{text-transform:lowercase!important}.is-uppercase{text-transform:uppercase!important}.is-italic{font-style:italic!important}.has-text-white{color:#fff!important}a.has-text-white:focus,a.has-text-white:hover{color:#e6e6e6!important}.has-background-white{background-color:#fff!important}.has-text-black{color:#0a0a0a!important}a.has-text-black:focus,a.has-text-black:hover{color:#000!important}.has-background-black{background-color:#0a0a0a!important}.has-text-light{color:#f5f5f5!important}a.has-text-light:focus,a.has-text-light:hover{color:#dbdbdb!important}.has-background-light{background-color:#f5f5f5!important}.has-text-dark{color:#363636!important}a.has-text-dark:focus,a.has-text-dark:hover{color:#1c1c1c!important}.has-background-dark{background-color:#363636!important}.has-text-primary{color:#00d1b2!important}a.has-text-primary:focus,a.has-text-primary:hover{color:#009e86!important}.has-background-primary{background-color:#00d1b2!important}.has-text-link{color:#3273dc!important}a.has-text-link:focus,a.has-text-link:hover{color:#205bbc!important}.has-background-link{background-color:#3273dc!important}.has-text-info{color:#3298dc!important}a.has-text-info:focus,a.has-text-info:hover{color:#207dbc!important}.has-background-info{background-color:#3298dc!important}.has-text-success{color:#48c774!important}a.has-text-success:focus,a.has-text-success:hover{color:#34a85c!important}.has-background-success{background-color:#48c774!important}.has-text-warning{color:#ffdd57!important}a.has-text-warning:focus,a.has-text-warning:hover{color:#ffd324!important}.has-background-warning{background-color:#ffdd57!important}.has-text-danger{color:#f14668!important}a.has-text-danger:focus,a.has-text-danger:hover{color:#ee1742!important}.has-background-danger{background-color:#f14668!important}.has-text-black-bis{color:#121212!important}.has-background-black-bis{background-color:#121212!important}.has-text-black-ter{color:#242424!important}.has-background-black-ter{background-color:#242424!important}.has-text-grey-darker{color:#363636!important}.has-background-grey-darker{background-color:#363636!important}.has-text-grey-dark{color:#4a4a4a!important}.has-background-grey-dark{background-color:#4a4a4a!important}.has-text-grey{color:#7a7a7a!important}.has-background-grey{background-color:#7a7a7a!important}.has-text-grey-light{color:#b5b5b5!important}.has-background-grey-light{background-color:#b5b5b5!important}.has-text-grey-lighter{color:#dbdbdb!important}.has-background-grey-lighter{background-color:#dbdbdb!important}.has-text-white-ter{color:#f5f5f5!important}.has-background-white-ter{background-color:#f5f5f5!important}.has-text-white-bis{color:#fafafa!important}.has-background-white-bis{background-color:#fafafa!important}.has-text-weight-light{font-weight:300!important}.has-text-weight-normal{font-weight:400!important}.has-text-weight-medium{font-weight:500!important}.has-text-weight-semibold{font-weight:600!important}.has-text-weight-bold{font-weight:700!important}.is-family-primary{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-secondary{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-sans-serif{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-monospace{font-family:monospace!important}.is-family-code{font-family:monospace!important}.is-block{display:block!important}@media screen and (max-width:768px){.is-block-mobile{display:block!important}}@media screen and (min-width:769px),print{.is-block-tablet{display:block!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-block-tablet-only{display:block!important}}@media screen and (max-width:1023px){.is-block-touch{display:block!important}}@media screen and (min-width:1024px){.is-block-desktop{display:block!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-block-desktop-only{display:block!important}}@media screen and (min-width:1216px){.is-block-widescreen{display:block!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-block-widescreen-only{display:block!important}}@media screen and (min-width:1408px){.is-block-fullhd{display:block!important}}.is-flex{display:flex!important}@media screen and (max-width:768px){.is-flex-mobile{display:flex!important}}@media screen and (min-width:769px),print{.is-flex-tablet{display:flex!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-flex-tablet-only{display:flex!important}}@media screen and (max-width:1023px){.is-flex-touch{display:flex!important}}@media screen and (min-width:1024px){.is-flex-desktop{display:flex!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-flex-desktop-only{display:flex!important}}@media screen and (min-width:1216px){.is-flex-widescreen{display:flex!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-flex-widescreen-only{display:flex!important}}@media screen and (min-width:1408px){.is-flex-fullhd{display:flex!important}}.is-inline{display:inline!important}@media screen and (max-width:768px){.is-inline-mobile{display:inline!important}}@media screen and (min-width:769px),print{.is-inline-tablet{display:inline!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-tablet-only{display:inline!important}}@media screen and (max-width:1023px){.is-inline-touch{display:inline!important}}@media screen and (min-width:1024px){.is-inline-desktop{display:inline!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-desktop-only{display:inline!important}}@media screen and (min-width:1216px){.is-inline-widescreen{display:inline!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-widescreen-only{display:inline!important}}@media screen and (min-width:1408px){.is-inline-fullhd{display:inline!important}}.is-inline-block{display:inline-block!important}@media screen and (max-width:768px){.is-inline-block-mobile{display:inline-block!important}}@media screen and (min-width:769px),print{.is-inline-block-tablet{display:inline-block!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-block-tablet-only{display:inline-block!important}}@media screen and (max-width:1023px){.is-inline-block-touch{display:inline-block!important}}@media screen and (min-width:1024px){.is-inline-block-desktop{display:inline-block!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-block-desktop-only{display:inline-block!important}}@media screen and (min-width:1216px){.is-inline-block-widescreen{display:inline-block!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-block-widescreen-only{display:inline-block!important}}@media screen and (min-width:1408px){.is-inline-block-fullhd{display:inline-block!important}}.is-inline-flex{display:inline-flex!important}@media screen and (max-width:768px){.is-inline-flex-mobile{display:inline-flex!important}}@media screen and (min-width:769px),print{.is-inline-flex-tablet{display:inline-flex!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-flex-tablet-only{display:inline-flex!important}}@media screen and (max-width:1023px){.is-inline-flex-touch{display:inline-flex!important}}@media screen and (min-width:1024px){.is-inline-flex-desktop{display:inline-flex!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-flex-desktop-only{display:inline-flex!important}}@media screen and (min-width:1216px){.is-inline-flex-widescreen{display:inline-flex!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-flex-widescreen-only{display:inline-flex!important}}@media screen and (min-width:1408px){.is-inline-flex-fullhd{display:inline-flex!important}}.is-hidden{display:none!important}.is-sr-only{border:none!important;clip:rect(0,0,0,0)!important;height:.01em!important;overflow:hidden!important;padding:0!important;position:absolute!important;white-space:nowrap!important;width:.01em!important}@media screen and (max-width:768px){.is-hidden-mobile{display:none!important}}@media screen and (min-width:769px),print{.is-hidden-tablet{display:none!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-hidden-tablet-only{display:none!important}}@media screen and (max-width:1023px){.is-hidden-touch{display:none!important}}@media screen and (min-width:1024px){.is-hidden-desktop{display:none!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-hidden-desktop-only{display:none!important}}@media screen and (min-width:1216px){.is-hidden-widescreen{display:none!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-hidden-widescreen-only{display:none!important}}@media screen and (min-width:1408px){.is-hidden-fullhd{display:none!important}}.is-invisible{visibility:hidden!important}@media screen and (max-width:768px){.is-invisible-mobile{visibility:hidden!important}}@media screen and (min-width:769px),print{.is-invisible-tablet{visibility:hidden!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-invisible-tablet-only{visibility:hidden!important}}@media screen and (max-width:1023px){.is-invisible-touch{visibility:hidden!important}}@media screen and (min-width:1024px){.is-invisible-desktop{visibility:hidden!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-invisible-desktop-only{visibility:hidden!important}}@media screen and (min-width:1216px){.is-invisible-widescreen{visibility:hidden!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-invisible-widescreen-only{visibility:hidden!important}}@media screen and (min-width:1408px){.is-invisible-fullhd{visibility:hidden!important}}.is-marginless{margin:0!important}.is-paddingless{padding:0!important}.is-radiusless{border-radius:0!important}.is-shadowless{box-shadow:none!important}.is-relative{position:relative!important}.box{background-color:#fff;border-radius:6px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);color:#4a4a4a;display:block;padding:1.25rem}a.box:focus,a.box:hover{box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px #3273dc}a.box:active{box-shadow:inset 0 1px 2px rgba(10,10,10,.2),0 0 0 1px #3273dc}.button{background-color:#fff;border-color:#dbdbdb;border-width:1px;color:#363636;cursor:pointer;justify-content:center;padding-bottom:calc(.5em - 1px);padding-left:1em;padding-right:1em;padding-top:calc(.5em - 1px);text-align:center;white-space:nowrap}.button strong{color:inherit}.button .icon,.button .icon.is-large,.button .icon.is-medium,.button .icon.is-small{height:1.5em;width:1.5em}.button .icon:first-child:not(:last-child){margin-left:calc(-.5em - 1px);margin-right:.25em}.button .icon:last-child:not(:first-child){margin-left:.25em;margin-right:calc(-.5em - 1px)}.button .icon:first-child:last-child{margin-left:calc(-.5em - 1px);margin-right:calc(-.5em - 1px)}.button.is-hovered,.button:hover{border-color:#b5b5b5;color:#363636}.button.is-focused,.button:focus{border-color:#3273dc;color:#363636}.button.is-focused:not(:active),.button:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.button.is-active,.button:active{border-color:#4a4a4a;color:#363636}.button.is-text{background-color:transparent;border-color:transparent;color:#4a4a4a;text-decoration:underline}.button.is-text.is-focused,.button.is-text.is-hovered,.button.is-text:focus,.button.is-text:hover{background-color:#f5f5f5;color:#363636}.button.is-text.is-active,.button.is-text:active{background-color:#e8e8e8;color:#363636}.button.is-text[disabled],fieldset[disabled] .button.is-text{background-color:transparent;border-color:transparent;box-shadow:none}.button.is-white{background-color:#fff;border-color:transparent;color:#0a0a0a}.button.is-white.is-hovered,.button.is-white:hover{background-color:#f9f9f9;border-color:transparent;color:#0a0a0a}.button.is-white.is-focused,.button.is-white:focus{border-color:transparent;color:#0a0a0a}.button.is-white.is-focused:not(:active),.button.is-white:focus:not(:active){box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.button.is-white.is-active,.button.is-white:active{background-color:#f2f2f2;border-color:transparent;color:#0a0a0a}.button.is-white[disabled],fieldset[disabled] .button.is-white{background-color:#fff;border-color:transparent;box-shadow:none}.button.is-white.is-inverted{background-color:#0a0a0a;color:#fff}.button.is-white.is-inverted.is-hovered,.button.is-white.is-inverted:hover{background-color:#000}.button.is-white.is-inverted[disabled],fieldset[disabled] .button.is-white.is-inverted{background-color:#0a0a0a;border-color:transparent;box-shadow:none;color:#fff}.button.is-white.is-loading::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-white.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-white.is-outlined.is-focused,.button.is-white.is-outlined.is-hovered,.button.is-white.is-outlined:focus,.button.is-white.is-outlined:hover{background-color:#fff;border-color:#fff;color:#0a0a0a}.button.is-white.is-outlined.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-white.is-outlined.is-loading.is-focused::after,.button.is-white.is-outlined.is-loading.is-hovered::after,.button.is-white.is-outlined.is-loading:focus::after,.button.is-white.is-outlined.is-loading:hover::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-white.is-outlined[disabled],fieldset[disabled] .button.is-white.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-white.is-inverted.is-outlined{background-color:transparent;border-color:#0a0a0a;color:#0a0a0a}.button.is-white.is-inverted.is-outlined.is-focused,.button.is-white.is-inverted.is-outlined.is-hovered,.button.is-white.is-inverted.is-outlined:focus,.button.is-white.is-inverted.is-outlined:hover{background-color:#0a0a0a;color:#fff}.button.is-white.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-white.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-white.is-inverted.is-outlined.is-loading:focus::after,.button.is-white.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-white.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-white.is-inverted.is-outlined{background-color:transparent;border-color:#0a0a0a;box-shadow:none;color:#0a0a0a}.button.is-black{background-color:#0a0a0a;border-color:transparent;color:#fff}.button.is-black.is-hovered,.button.is-black:hover{background-color:#040404;border-color:transparent;color:#fff}.button.is-black.is-focused,.button.is-black:focus{border-color:transparent;color:#fff}.button.is-black.is-focused:not(:active),.button.is-black:focus:not(:active){box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.button.is-black.is-active,.button.is-black:active{background-color:#000;border-color:transparent;color:#fff}.button.is-black[disabled],fieldset[disabled] .button.is-black{background-color:#0a0a0a;border-color:transparent;box-shadow:none}.button.is-black.is-inverted{background-color:#fff;color:#0a0a0a}.button.is-black.is-inverted.is-hovered,.button.is-black.is-inverted:hover{background-color:#f2f2f2}.button.is-black.is-inverted[disabled],fieldset[disabled] .button.is-black.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#0a0a0a}.button.is-black.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-black.is-outlined{background-color:transparent;border-color:#0a0a0a;color:#0a0a0a}.button.is-black.is-outlined.is-focused,.button.is-black.is-outlined.is-hovered,.button.is-black.is-outlined:focus,.button.is-black.is-outlined:hover{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.button.is-black.is-outlined.is-loading::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-black.is-outlined.is-loading.is-focused::after,.button.is-black.is-outlined.is-loading.is-hovered::after,.button.is-black.is-outlined.is-loading:focus::after,.button.is-black.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-black.is-outlined[disabled],fieldset[disabled] .button.is-black.is-outlined{background-color:transparent;border-color:#0a0a0a;box-shadow:none;color:#0a0a0a}.button.is-black.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-black.is-inverted.is-outlined.is-focused,.button.is-black.is-inverted.is-outlined.is-hovered,.button.is-black.is-inverted.is-outlined:focus,.button.is-black.is-inverted.is-outlined:hover{background-color:#fff;color:#0a0a0a}.button.is-black.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-black.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-black.is-inverted.is-outlined.is-loading:focus::after,.button.is-black.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-black.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-black.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-light{background-color:#f5f5f5;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-hovered,.button.is-light:hover{background-color:#eee;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-focused,.button.is-light:focus{border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-focused:not(:active),.button.is-light:focus:not(:active){box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.button.is-light.is-active,.button.is-light:active{background-color:#e8e8e8;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light[disabled],fieldset[disabled] .button.is-light{background-color:#f5f5f5;border-color:transparent;box-shadow:none}.button.is-light.is-inverted{background-color:rgba(0,0,0,.7);color:#f5f5f5}.button.is-light.is-inverted.is-hovered,.button.is-light.is-inverted:hover{background-color:rgba(0,0,0,.7)}.button.is-light.is-inverted[disabled],fieldset[disabled] .button.is-light.is-inverted{background-color:rgba(0,0,0,.7);border-color:transparent;box-shadow:none;color:#f5f5f5}.button.is-light.is-loading::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-light.is-outlined{background-color:transparent;border-color:#f5f5f5;color:#f5f5f5}.button.is-light.is-outlined.is-focused,.button.is-light.is-outlined.is-hovered,.button.is-light.is-outlined:focus,.button.is-light.is-outlined:hover{background-color:#f5f5f5;border-color:#f5f5f5;color:rgba(0,0,0,.7)}.button.is-light.is-outlined.is-loading::after{border-color:transparent transparent #f5f5f5 #f5f5f5!important}.button.is-light.is-outlined.is-loading.is-focused::after,.button.is-light.is-outlined.is-loading.is-hovered::after,.button.is-light.is-outlined.is-loading:focus::after,.button.is-light.is-outlined.is-loading:hover::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-light.is-outlined[disabled],fieldset[disabled] .button.is-light.is-outlined{background-color:transparent;border-color:#f5f5f5;box-shadow:none;color:#f5f5f5}.button.is-light.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);color:rgba(0,0,0,.7)}.button.is-light.is-inverted.is-outlined.is-focused,.button.is-light.is-inverted.is-outlined.is-hovered,.button.is-light.is-inverted.is-outlined:focus,.button.is-light.is-inverted.is-outlined:hover{background-color:rgba(0,0,0,.7);color:#f5f5f5}.button.is-light.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-light.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-light.is-inverted.is-outlined.is-loading:focus::after,.button.is-light.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #f5f5f5 #f5f5f5!important}.button.is-light.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-light.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);box-shadow:none;color:rgba(0,0,0,.7)}.button.is-dark{background-color:#363636;border-color:transparent;color:#fff}.button.is-dark.is-hovered,.button.is-dark:hover{background-color:#2f2f2f;border-color:transparent;color:#fff}.button.is-dark.is-focused,.button.is-dark:focus{border-color:transparent;color:#fff}.button.is-dark.is-focused:not(:active),.button.is-dark:focus:not(:active){box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.button.is-dark.is-active,.button.is-dark:active{background-color:#292929;border-color:transparent;color:#fff}.button.is-dark[disabled],fieldset[disabled] .button.is-dark{background-color:#363636;border-color:transparent;box-shadow:none}.button.is-dark.is-inverted{background-color:#fff;color:#363636}.button.is-dark.is-inverted.is-hovered,.button.is-dark.is-inverted:hover{background-color:#f2f2f2}.button.is-dark.is-inverted[disabled],fieldset[disabled] .button.is-dark.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#363636}.button.is-dark.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-dark.is-outlined{background-color:transparent;border-color:#363636;color:#363636}.button.is-dark.is-outlined.is-focused,.button.is-dark.is-outlined.is-hovered,.button.is-dark.is-outlined:focus,.button.is-dark.is-outlined:hover{background-color:#363636;border-color:#363636;color:#fff}.button.is-dark.is-outlined.is-loading::after{border-color:transparent transparent #363636 #363636!important}.button.is-dark.is-outlined.is-loading.is-focused::after,.button.is-dark.is-outlined.is-loading.is-hovered::after,.button.is-dark.is-outlined.is-loading:focus::after,.button.is-dark.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-dark.is-outlined[disabled],fieldset[disabled] .button.is-dark.is-outlined{background-color:transparent;border-color:#363636;box-shadow:none;color:#363636}.button.is-dark.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-dark.is-inverted.is-outlined.is-focused,.button.is-dark.is-inverted.is-outlined.is-hovered,.button.is-dark.is-inverted.is-outlined:focus,.button.is-dark.is-inverted.is-outlined:hover{background-color:#fff;color:#363636}.button.is-dark.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-dark.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-dark.is-inverted.is-outlined.is-loading:focus::after,.button.is-dark.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #363636 #363636!important}.button.is-dark.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-dark.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-primary{background-color:#00d1b2;border-color:transparent;color:#fff}.button.is-primary.is-hovered,.button.is-primary:hover{background-color:#00c4a7;border-color:transparent;color:#fff}.button.is-primary.is-focused,.button.is-primary:focus{border-color:transparent;color:#fff}.button.is-primary.is-focused:not(:active),.button.is-primary:focus:not(:active){box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.button.is-primary.is-active,.button.is-primary:active{background-color:#00b89c;border-color:transparent;color:#fff}.button.is-primary[disabled],fieldset[disabled] .button.is-primary{background-color:#00d1b2;border-color:transparent;box-shadow:none}.button.is-primary.is-inverted{background-color:#fff;color:#00d1b2}.button.is-primary.is-inverted.is-hovered,.button.is-primary.is-inverted:hover{background-color:#f2f2f2}.button.is-primary.is-inverted[disabled],fieldset[disabled] .button.is-primary.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#00d1b2}.button.is-primary.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-primary.is-outlined{background-color:transparent;border-color:#00d1b2;color:#00d1b2}.button.is-primary.is-outlined.is-focused,.button.is-primary.is-outlined.is-hovered,.button.is-primary.is-outlined:focus,.button.is-primary.is-outlined:hover{background-color:#00d1b2;border-color:#00d1b2;color:#fff}.button.is-primary.is-outlined.is-loading::after{border-color:transparent transparent #00d1b2 #00d1b2!important}.button.is-primary.is-outlined.is-loading.is-focused::after,.button.is-primary.is-outlined.is-loading.is-hovered::after,.button.is-primary.is-outlined.is-loading:focus::after,.button.is-primary.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-primary.is-outlined[disabled],fieldset[disabled] .button.is-primary.is-outlined{background-color:transparent;border-color:#00d1b2;box-shadow:none;color:#00d1b2}.button.is-primary.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-primary.is-inverted.is-outlined.is-focused,.button.is-primary.is-inverted.is-outlined.is-hovered,.button.is-primary.is-inverted.is-outlined:focus,.button.is-primary.is-inverted.is-outlined:hover{background-color:#fff;color:#00d1b2}.button.is-primary.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-primary.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-primary.is-inverted.is-outlined.is-loading:focus::after,.button.is-primary.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #00d1b2 #00d1b2!important}.button.is-primary.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-primary.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-primary.is-light{background-color:#ebfffc;color:#00947e}.button.is-primary.is-light.is-hovered,.button.is-primary.is-light:hover{background-color:#defffa;border-color:transparent;color:#00947e}.button.is-primary.is-light.is-active,.button.is-primary.is-light:active{background-color:#d1fff8;border-color:transparent;color:#00947e}.button.is-link{background-color:#3273dc;border-color:transparent;color:#fff}.button.is-link.is-hovered,.button.is-link:hover{background-color:#276cda;border-color:transparent;color:#fff}.button.is-link.is-focused,.button.is-link:focus{border-color:transparent;color:#fff}.button.is-link.is-focused:not(:active),.button.is-link:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.button.is-link.is-active,.button.is-link:active{background-color:#2366d1;border-color:transparent;color:#fff}.button.is-link[disabled],fieldset[disabled] .button.is-link{background-color:#3273dc;border-color:transparent;box-shadow:none}.button.is-link.is-inverted{background-color:#fff;color:#3273dc}.button.is-link.is-inverted.is-hovered,.button.is-link.is-inverted:hover{background-color:#f2f2f2}.button.is-link.is-inverted[disabled],fieldset[disabled] .button.is-link.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#3273dc}.button.is-link.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-link.is-outlined{background-color:transparent;border-color:#3273dc;color:#3273dc}.button.is-link.is-outlined.is-focused,.button.is-link.is-outlined.is-hovered,.button.is-link.is-outlined:focus,.button.is-link.is-outlined:hover{background-color:#3273dc;border-color:#3273dc;color:#fff}.button.is-link.is-outlined.is-loading::after{border-color:transparent transparent #3273dc #3273dc!important}.button.is-link.is-outlined.is-loading.is-focused::after,.button.is-link.is-outlined.is-loading.is-hovered::after,.button.is-link.is-outlined.is-loading:focus::after,.button.is-link.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-link.is-outlined[disabled],fieldset[disabled] .button.is-link.is-outlined{background-color:transparent;border-color:#3273dc;box-shadow:none;color:#3273dc}.button.is-link.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-link.is-inverted.is-outlined.is-focused,.button.is-link.is-inverted.is-outlined.is-hovered,.button.is-link.is-inverted.is-outlined:focus,.button.is-link.is-inverted.is-outlined:hover{background-color:#fff;color:#3273dc}.button.is-link.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-link.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-link.is-inverted.is-outlined.is-loading:focus::after,.button.is-link.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #3273dc #3273dc!important}.button.is-link.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-link.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-link.is-light{background-color:#eef3fc;color:#2160c4}.button.is-link.is-light.is-hovered,.button.is-link.is-light:hover{background-color:#e3ecfa;border-color:transparent;color:#2160c4}.button.is-link.is-light.is-active,.button.is-link.is-light:active{background-color:#d8e4f8;border-color:transparent;color:#2160c4}.button.is-info{background-color:#3298dc;border-color:transparent;color:#fff}.button.is-info.is-hovered,.button.is-info:hover{background-color:#2793da;border-color:transparent;color:#fff}.button.is-info.is-focused,.button.is-info:focus{border-color:transparent;color:#fff}.button.is-info.is-focused:not(:active),.button.is-info:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.button.is-info.is-active,.button.is-info:active{background-color:#238cd1;border-color:transparent;color:#fff}.button.is-info[disabled],fieldset[disabled] .button.is-info{background-color:#3298dc;border-color:transparent;box-shadow:none}.button.is-info.is-inverted{background-color:#fff;color:#3298dc}.button.is-info.is-inverted.is-hovered,.button.is-info.is-inverted:hover{background-color:#f2f2f2}.button.is-info.is-inverted[disabled],fieldset[disabled] .button.is-info.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#3298dc}.button.is-info.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-info.is-outlined{background-color:transparent;border-color:#3298dc;color:#3298dc}.button.is-info.is-outlined.is-focused,.button.is-info.is-outlined.is-hovered,.button.is-info.is-outlined:focus,.button.is-info.is-outlined:hover{background-color:#3298dc;border-color:#3298dc;color:#fff}.button.is-info.is-outlined.is-loading::after{border-color:transparent transparent #3298dc #3298dc!important}.button.is-info.is-outlined.is-loading.is-focused::after,.button.is-info.is-outlined.is-loading.is-hovered::after,.button.is-info.is-outlined.is-loading:focus::after,.button.is-info.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-info.is-outlined[disabled],fieldset[disabled] .button.is-info.is-outlined{background-color:transparent;border-color:#3298dc;box-shadow:none;color:#3298dc}.button.is-info.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-info.is-inverted.is-outlined.is-focused,.button.is-info.is-inverted.is-outlined.is-hovered,.button.is-info.is-inverted.is-outlined:focus,.button.is-info.is-inverted.is-outlined:hover{background-color:#fff;color:#3298dc}.button.is-info.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-info.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-info.is-inverted.is-outlined.is-loading:focus::after,.button.is-info.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #3298dc #3298dc!important}.button.is-info.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-info.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-info.is-light{background-color:#eef6fc;color:#1d72aa}.button.is-info.is-light.is-hovered,.button.is-info.is-light:hover{background-color:#e3f1fa;border-color:transparent;color:#1d72aa}.button.is-info.is-light.is-active,.button.is-info.is-light:active{background-color:#d8ebf8;border-color:transparent;color:#1d72aa}.button.is-success{background-color:#48c774;border-color:transparent;color:#fff}.button.is-success.is-hovered,.button.is-success:hover{background-color:#3ec46d;border-color:transparent;color:#fff}.button.is-success.is-focused,.button.is-success:focus{border-color:transparent;color:#fff}.button.is-success.is-focused:not(:active),.button.is-success:focus:not(:active){box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.button.is-success.is-active,.button.is-success:active{background-color:#3abb67;border-color:transparent;color:#fff}.button.is-success[disabled],fieldset[disabled] .button.is-success{background-color:#48c774;border-color:transparent;box-shadow:none}.button.is-success.is-inverted{background-color:#fff;color:#48c774}.button.is-success.is-inverted.is-hovered,.button.is-success.is-inverted:hover{background-color:#f2f2f2}.button.is-success.is-inverted[disabled],fieldset[disabled] .button.is-success.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#48c774}.button.is-success.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-success.is-outlined{background-color:transparent;border-color:#48c774;color:#48c774}.button.is-success.is-outlined.is-focused,.button.is-success.is-outlined.is-hovered,.button.is-success.is-outlined:focus,.button.is-success.is-outlined:hover{background-color:#48c774;border-color:#48c774;color:#fff}.button.is-success.is-outlined.is-loading::after{border-color:transparent transparent #48c774 #48c774!important}.button.is-success.is-outlined.is-loading.is-focused::after,.button.is-success.is-outlined.is-loading.is-hovered::after,.button.is-success.is-outlined.is-loading:focus::after,.button.is-success.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-success.is-outlined[disabled],fieldset[disabled] .button.is-success.is-outlined{background-color:transparent;border-color:#48c774;box-shadow:none;color:#48c774}.button.is-success.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-success.is-inverted.is-outlined.is-focused,.button.is-success.is-inverted.is-outlined.is-hovered,.button.is-success.is-inverted.is-outlined:focus,.button.is-success.is-inverted.is-outlined:hover{background-color:#fff;color:#48c774}.button.is-success.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-success.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-success.is-inverted.is-outlined.is-loading:focus::after,.button.is-success.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #48c774 #48c774!important}.button.is-success.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-success.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-success.is-light{background-color:#effaf3;color:#257942}.button.is-success.is-light.is-hovered,.button.is-success.is-light:hover{background-color:#e6f7ec;border-color:transparent;color:#257942}.button.is-success.is-light.is-active,.button.is-success.is-light:active{background-color:#dcf4e4;border-color:transparent;color:#257942}.button.is-warning{background-color:#ffdd57;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-hovered,.button.is-warning:hover{background-color:#ffdb4a;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-focused,.button.is-warning:focus{border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-focused:not(:active),.button.is-warning:focus:not(:active){box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.button.is-warning.is-active,.button.is-warning:active{background-color:#ffd83d;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning[disabled],fieldset[disabled] .button.is-warning{background-color:#ffdd57;border-color:transparent;box-shadow:none}.button.is-warning.is-inverted{background-color:rgba(0,0,0,.7);color:#ffdd57}.button.is-warning.is-inverted.is-hovered,.button.is-warning.is-inverted:hover{background-color:rgba(0,0,0,.7)}.button.is-warning.is-inverted[disabled],fieldset[disabled] .button.is-warning.is-inverted{background-color:rgba(0,0,0,.7);border-color:transparent;box-shadow:none;color:#ffdd57}.button.is-warning.is-loading::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-warning.is-outlined{background-color:transparent;border-color:#ffdd57;color:#ffdd57}.button.is-warning.is-outlined.is-focused,.button.is-warning.is-outlined.is-hovered,.button.is-warning.is-outlined:focus,.button.is-warning.is-outlined:hover{background-color:#ffdd57;border-color:#ffdd57;color:rgba(0,0,0,.7)}.button.is-warning.is-outlined.is-loading::after{border-color:transparent transparent #ffdd57 #ffdd57!important}.button.is-warning.is-outlined.is-loading.is-focused::after,.button.is-warning.is-outlined.is-loading.is-hovered::after,.button.is-warning.is-outlined.is-loading:focus::after,.button.is-warning.is-outlined.is-loading:hover::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-warning.is-outlined[disabled],fieldset[disabled] .button.is-warning.is-outlined{background-color:transparent;border-color:#ffdd57;box-shadow:none;color:#ffdd57}.button.is-warning.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);color:rgba(0,0,0,.7)}.button.is-warning.is-inverted.is-outlined.is-focused,.button.is-warning.is-inverted.is-outlined.is-hovered,.button.is-warning.is-inverted.is-outlined:focus,.button.is-warning.is-inverted.is-outlined:hover{background-color:rgba(0,0,0,.7);color:#ffdd57}.button.is-warning.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-warning.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-warning.is-inverted.is-outlined.is-loading:focus::after,.button.is-warning.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #ffdd57 #ffdd57!important}.button.is-warning.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-warning.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);box-shadow:none;color:rgba(0,0,0,.7)}.button.is-warning.is-light{background-color:#fffbeb;color:#947600}.button.is-warning.is-light.is-hovered,.button.is-warning.is-light:hover{background-color:#fff8de;border-color:transparent;color:#947600}.button.is-warning.is-light.is-active,.button.is-warning.is-light:active{background-color:#fff6d1;border-color:transparent;color:#947600}.button.is-danger{background-color:#f14668;border-color:transparent;color:#fff}.button.is-danger.is-hovered,.button.is-danger:hover{background-color:#f03a5f;border-color:transparent;color:#fff}.button.is-danger.is-focused,.button.is-danger:focus{border-color:transparent;color:#fff}.button.is-danger.is-focused:not(:active),.button.is-danger:focus:not(:active){box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.button.is-danger.is-active,.button.is-danger:active{background-color:#ef2e55;border-color:transparent;color:#fff}.button.is-danger[disabled],fieldset[disabled] .button.is-danger{background-color:#f14668;border-color:transparent;box-shadow:none}.button.is-danger.is-inverted{background-color:#fff;color:#f14668}.button.is-danger.is-inverted.is-hovered,.button.is-danger.is-inverted:hover{background-color:#f2f2f2}.button.is-danger.is-inverted[disabled],fieldset[disabled] .button.is-danger.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#f14668}.button.is-danger.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-danger.is-outlined{background-color:transparent;border-color:#f14668;color:#f14668}.button.is-danger.is-outlined.is-focused,.button.is-danger.is-outlined.is-hovered,.button.is-danger.is-outlined:focus,.button.is-danger.is-outlined:hover{background-color:#f14668;border-color:#f14668;color:#fff}.button.is-danger.is-outlined.is-loading::after{border-color:transparent transparent #f14668 #f14668!important}.button.is-danger.is-outlined.is-loading.is-focused::after,.button.is-danger.is-outlined.is-loading.is-hovered::after,.button.is-danger.is-outlined.is-loading:focus::after,.button.is-danger.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-danger.is-outlined[disabled],fieldset[disabled] .button.is-danger.is-outlined{background-color:transparent;border-color:#f14668;box-shadow:none;color:#f14668}.button.is-danger.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-danger.is-inverted.is-outlined.is-focused,.button.is-danger.is-inverted.is-outlined.is-hovered,.button.is-danger.is-inverted.is-outlined:focus,.button.is-danger.is-inverted.is-outlined:hover{background-color:#fff;color:#f14668}.button.is-danger.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-danger.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-danger.is-inverted.is-outlined.is-loading:focus::after,.button.is-danger.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #f14668 #f14668!important}.button.is-danger.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-danger.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-danger.is-light{background-color:#feecf0;color:#cc0f35}.button.is-danger.is-light.is-hovered,.button.is-danger.is-light:hover{background-color:#fde0e6;border-color:transparent;color:#cc0f35}.button.is-danger.is-light.is-active,.button.is-danger.is-light:active{background-color:#fcd4dc;border-color:transparent;color:#cc0f35}.button.is-small{border-radius:2px;font-size:.75rem}.button.is-normal{font-size:1rem}.button.is-medium{font-size:1.25rem}.button.is-large{font-size:1.5rem}.button[disabled],fieldset[disabled] .button{background-color:#fff;border-color:#dbdbdb;box-shadow:none;opacity:.5}.button.is-fullwidth{display:flex;width:100%}.button.is-loading{color:transparent!important;pointer-events:none}.button.is-loading::after{position:absolute;left:calc(50% - (1em / 2));top:calc(50% - (1em / 2));position:absolute!important}.button.is-static{background-color:#f5f5f5;border-color:#dbdbdb;color:#7a7a7a;box-shadow:none;pointer-events:none}.button.is-rounded{border-radius:290486px;padding-left:calc(1em + .25em);padding-right:calc(1em + .25em)}.buttons{align-items:center;display:flex;flex-wrap:wrap;justify-content:flex-start}.buttons .button{margin-bottom:.5rem}.buttons .button:not(:last-child):not(.is-fullwidth){margin-right:.5rem}.buttons:last-child{margin-bottom:-.5rem}.buttons:not(:last-child){margin-bottom:1rem}.buttons.are-small .button:not(.is-normal):not(.is-medium):not(.is-large){border-radius:2px;font-size:.75rem}.buttons.are-medium .button:not(.is-small):not(.is-normal):not(.is-large){font-size:1.25rem}.buttons.are-large .button:not(.is-small):not(.is-normal):not(.is-medium){font-size:1.5rem}.buttons.has-addons .button:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.buttons.has-addons .button:not(:last-child){border-bottom-right-radius:0;border-top-right-radius:0;margin-right:-1px}.buttons.has-addons .button:last-child{margin-right:0}.buttons.has-addons .button.is-hovered,.buttons.has-addons .button:hover{z-index:2}.buttons.has-addons .button.is-active,.buttons.has-addons .button.is-focused,.buttons.has-addons .button.is-selected,.buttons.has-addons .button:active,.buttons.has-addons .button:focus{z-index:3}.buttons.has-addons .button.is-active:hover,.buttons.has-addons .button.is-focused:hover,.buttons.has-addons .button.is-selected:hover,.buttons.has-addons .button:active:hover,.buttons.has-addons .button:focus:hover{z-index:4}.buttons.has-addons .button.is-expanded{flex-grow:1;flex-shrink:1}.buttons.is-centered{justify-content:center}.buttons.is-centered:not(.has-addons) .button:not(.is-fullwidth){margin-left:.25rem;margin-right:.25rem}.buttons.is-right{justify-content:flex-end}.buttons.is-right:not(.has-addons) .button:not(.is-fullwidth){margin-left:.25rem;margin-right:.25rem}.container{flex-grow:1;margin:0 auto;position:relative;width:auto}.container.is-fluid{max-width:none;padding-left:32px;padding-right:32px;width:100%}@media screen and (min-width:1024px){.container{max-width:960px}}@media screen and (max-width:1215px){.container.is-widescreen{max-width:1152px}}@media screen and (max-width:1407px){.container.is-fullhd{max-width:1344px}}@media screen and (min-width:1216px){.container{max-width:1152px}}@media screen and (min-width:1408px){.container{max-width:1344px}}.content li+li{margin-top:.25em}.content blockquote:not(:last-child),.content dl:not(:last-child),.content ol:not(:last-child),.content p:not(:last-child),.content pre:not(:last-child),.content table:not(:last-child),.content ul:not(:last-child){margin-bottom:1em}.content h1,.content h2,.content h3,.content h4,.content h5,.content h6{color:#363636;font-weight:600;line-height:1.125}.content h1{font-size:2em;margin-bottom:.5em}.content h1:not(:first-child){margin-top:1em}.content h2{font-size:1.75em;margin-bottom:.5714em}.content h2:not(:first-child){margin-top:1.1428em}.content h3{font-size:1.5em;margin-bottom:.6666em}.content h3:not(:first-child){margin-top:1.3333em}.content h4{font-size:1.25em;margin-bottom:.8em}.content h5{font-size:1.125em;margin-bottom:.8888em}.content h6{font-size:1em;margin-bottom:1em}.content blockquote{background-color:#f5f5f5;border-left:5px solid #dbdbdb;padding:1.25em 1.5em}.content ol{list-style-position:outside;margin-left:2em;margin-top:1em}.content ol:not([type]){list-style-type:decimal}.content ol:not([type]).is-lower-alpha{list-style-type:lower-alpha}.content ol:not([type]).is-lower-roman{list-style-type:lower-roman}.content ol:not([type]).is-upper-alpha{list-style-type:upper-alpha}.content ol:not([type]).is-upper-roman{list-style-type:upper-roman}.content ul{list-style:disc outside;margin-left:2em;margin-top:1em}.content ul ul{list-style-type:circle;margin-top:.5em}.content ul ul ul{list-style-type:square}.content dd{margin-left:2em}.content figure{margin-left:2em;margin-right:2em;text-align:center}.content figure:not(:first-child){margin-top:2em}.content figure:not(:last-child){margin-bottom:2em}.content figure img{display:inline-block}.content figure figcaption{font-style:italic}.content pre{-webkit-overflow-scrolling:touch;overflow-x:auto;padding:1.25em 1.5em;white-space:pre;word-wrap:normal}.content sub,.content sup{font-size:75%}.content table{width:100%}.content table td,.content table th{border:1px solid #dbdbdb;border-width:0 0 1px;padding:.5em .75em;vertical-align:top}.content table th{color:#363636}.content table th:not([align]){text-align:left}.content table thead td,.content table thead th{border-width:0 0 2px;color:#363636}.content table tfoot td,.content table tfoot th{border-width:2px 0 0;color:#363636}.content table tbody tr:last-child td,.content table tbody tr:last-child th{border-bottom-width:0}.content .tabs li+li{margin-top:0}.content.is-small{font-size:.75rem}.content.is-medium{font-size:1.25rem}.content.is-large{font-size:1.5rem}.icon{align-items:center;display:inline-flex;justify-content:center;height:1.5rem;width:1.5rem}.icon.is-small{height:1rem;width:1rem}.icon.is-medium{height:2rem;width:2rem}.icon.is-large{height:3rem;width:3rem}.image{display:block;position:relative}.image img{display:block;height:auto;width:100%}.image img.is-rounded{border-radius:290486px}.image.is-fullwidth{width:100%}.image.is-16by9 .has-ratio,.image.is-16by9 img,.image.is-1by1 .has-ratio,.image.is-1by1 img,.image.is-1by2 .has-ratio,.image.is-1by2 img,.image.is-1by3 .has-ratio,.image.is-1by3 img,.image.is-2by1 .has-ratio,.image.is-2by1 img,.image.is-2by3 .has-ratio,.image.is-2by3 img,.image.is-3by1 .has-ratio,.image.is-3by1 img,.image.is-3by2 .has-ratio,.image.is-3by2 img,.image.is-3by4 .has-ratio,.image.is-3by4 img,.image.is-3by5 .has-ratio,.image.is-3by5 img,.image.is-4by3 .has-ratio,.image.is-4by3 img,.image.is-4by5 .has-ratio,.image.is-4by5 img,.image.is-5by3 .has-ratio,.image.is-5by3 img,.image.is-5by4 .has-ratio,.image.is-5by4 img,.image.is-9by16 .has-ratio,.image.is-9by16 img,.image.is-square .has-ratio,.image.is-square img{height:100%;width:100%}.image.is-1by1,.image.is-square{padding-top:100%}.image.is-5by4{padding-top:80%}.image.is-4by3{padding-top:75%}.image.is-3by2{padding-top:66.6666%}.image.is-5by3{padding-top:60%}.image.is-16by9{padding-top:56.25%}.image.is-2by1{padding-top:50%}.image.is-3by1{padding-top:33.3333%}.image.is-4by5{padding-top:125%}.image.is-3by4{padding-top:133.3333%}.image.is-2by3{padding-top:150%}.image.is-3by5{padding-top:166.6666%}.image.is-9by16{padding-top:177.7777%}.image.is-1by2{padding-top:200%}.image.is-1by3{padding-top:300%}.image.is-16x16{height:16px;width:16px}.image.is-24x24{height:24px;width:24px}.image.is-32x32{height:32px;width:32px}.image.is-48x48{height:48px;width:48px}.image.is-64x64{height:64px;width:64px}.image.is-96x96{height:96px;width:96px}.image.is-128x128{height:128px;width:128px}.notification{background-color:#f5f5f5;border-radius:4px;padding:1.25rem 2.5rem 1.25rem 1.5rem;position:relative}.notification a:not(.button):not(.dropdown-item){color:currentColor;text-decoration:underline}.notification strong{color:currentColor}.notification code,.notification pre{background:#fff}.notification pre code{background:0 0}.notification>.delete{position:absolute;right:.5rem;top:.5rem}.notification .content,.notification .subtitle,.notification .title{color:currentColor}.notification.is-white{background-color:#fff;color:#0a0a0a}.notification.is-black{background-color:#0a0a0a;color:#fff}.notification.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.notification.is-dark{background-color:#363636;color:#fff}.notification.is-primary{background-color:#00d1b2;color:#fff}.notification.is-link{background-color:#3273dc;color:#fff}.notification.is-info{background-color:#3298dc;color:#fff}.notification.is-success{background-color:#48c774;color:#fff}.notification.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.notification.is-danger{background-color:#f14668;color:#fff}.progress{-moz-appearance:none;-webkit-appearance:none;border:none;border-radius:290486px;display:block;height:1rem;overflow:hidden;padding:0;width:100%}.progress::-webkit-progress-bar{background-color:#ededed}.progress::-webkit-progress-value{background-color:#4a4a4a}.progress::-moz-progress-bar{background-color:#4a4a4a}.progress::-ms-fill{background-color:#4a4a4a;border:none}.progress.is-white::-webkit-progress-value{background-color:#fff}.progress.is-white::-moz-progress-bar{background-color:#fff}.progress.is-white::-ms-fill{background-color:#fff}.progress.is-white:indeterminate{background-image:linear-gradient(to right,#fff 30%,#ededed 30%)}.progress.is-black::-webkit-progress-value{background-color:#0a0a0a}.progress.is-black::-moz-progress-bar{background-color:#0a0a0a}.progress.is-black::-ms-fill{background-color:#0a0a0a}.progress.is-black:indeterminate{background-image:linear-gradient(to right,#0a0a0a 30%,#ededed 30%)}.progress.is-light::-webkit-progress-value{background-color:#f5f5f5}.progress.is-light::-moz-progress-bar{background-color:#f5f5f5}.progress.is-light::-ms-fill{background-color:#f5f5f5}.progress.is-light:indeterminate{background-image:linear-gradient(to right,#f5f5f5 30%,#ededed 30%)}.progress.is-dark::-webkit-progress-value{background-color:#363636}.progress.is-dark::-moz-progress-bar{background-color:#363636}.progress.is-dark::-ms-fill{background-color:#363636}.progress.is-dark:indeterminate{background-image:linear-gradient(to right,#363636 30%,#ededed 30%)}.progress.is-primary::-webkit-progress-value{background-color:#00d1b2}.progress.is-primary::-moz-progress-bar{background-color:#00d1b2}.progress.is-primary::-ms-fill{background-color:#00d1b2}.progress.is-primary:indeterminate{background-image:linear-gradient(to right,#00d1b2 30%,#ededed 30%)}.progress.is-link::-webkit-progress-value{background-color:#3273dc}.progress.is-link::-moz-progress-bar{background-color:#3273dc}.progress.is-link::-ms-fill{background-color:#3273dc}.progress.is-link:indeterminate{background-image:linear-gradient(to right,#3273dc 30%,#ededed 30%)}.progress.is-info::-webkit-progress-value{background-color:#3298dc}.progress.is-info::-moz-progress-bar{background-color:#3298dc}.progress.is-info::-ms-fill{background-color:#3298dc}.progress.is-info:indeterminate{background-image:linear-gradient(to right,#3298dc 30%,#ededed 30%)}.progress.is-success::-webkit-progress-value{background-color:#48c774}.progress.is-success::-moz-progress-bar{background-color:#48c774}.progress.is-success::-ms-fill{background-color:#48c774}.progress.is-success:indeterminate{background-image:linear-gradient(to right,#48c774 30%,#ededed 30%)}.progress.is-warning::-webkit-progress-value{background-color:#ffdd57}.progress.is-warning::-moz-progress-bar{background-color:#ffdd57}.progress.is-warning::-ms-fill{background-color:#ffdd57}.progress.is-warning:indeterminate{background-image:linear-gradient(to right,#ffdd57 30%,#ededed 30%)}.progress.is-danger::-webkit-progress-value{background-color:#f14668}.progress.is-danger::-moz-progress-bar{background-color:#f14668}.progress.is-danger::-ms-fill{background-color:#f14668}.progress.is-danger:indeterminate{background-image:linear-gradient(to right,#f14668 30%,#ededed 30%)}.progress:indeterminate{-webkit-animation-duration:1.5s;animation-duration:1.5s;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;-webkit-animation-name:moveIndeterminate;animation-name:moveIndeterminate;-webkit-animation-timing-function:linear;animation-timing-function:linear;background-color:#ededed;background-image:linear-gradient(to right,#4a4a4a 30%,#ededed 30%);background-position:top left;background-repeat:no-repeat;background-size:150% 150%}.progress:indeterminate::-webkit-progress-bar{background-color:transparent}.progress:indeterminate::-moz-progress-bar{background-color:transparent}.progress.is-small{height:.75rem}.progress.is-medium{height:1.25rem}.progress.is-large{height:1.5rem}@-webkit-keyframes moveIndeterminate{from{background-position:200% 0}to{background-position:-200% 0}}@keyframes moveIndeterminate{from{background-position:200% 0}to{background-position:-200% 0}}.table{background-color:#fff;color:#363636}.table td,.table th{border:1px solid #dbdbdb;border-width:0 0 1px;padding:.5em .75em;vertical-align:top}.table td.is-white,.table th.is-white{background-color:#fff;border-color:#fff;color:#0a0a0a}.table td.is-black,.table th.is-black{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.table td.is-light,.table th.is-light{background-color:#f5f5f5;border-color:#f5f5f5;color:rgba(0,0,0,.7)}.table td.is-dark,.table th.is-dark{background-color:#363636;border-color:#363636;color:#fff}.table td.is-primary,.table th.is-primary{background-color:#00d1b2;border-color:#00d1b2;color:#fff}.table td.is-link,.table th.is-link{background-color:#3273dc;border-color:#3273dc;color:#fff}.table td.is-info,.table th.is-info{background-color:#3298dc;border-color:#3298dc;color:#fff}.table td.is-success,.table th.is-success{background-color:#48c774;border-color:#48c774;color:#fff}.table td.is-warning,.table th.is-warning{background-color:#ffdd57;border-color:#ffdd57;color:rgba(0,0,0,.7)}.table td.is-danger,.table th.is-danger{background-color:#f14668;border-color:#f14668;color:#fff}.table td.is-narrow,.table th.is-narrow{white-space:nowrap;width:1%}.table td.is-selected,.table th.is-selected{background-color:#00d1b2;color:#fff}.table td.is-selected a,.table td.is-selected strong,.table th.is-selected a,.table th.is-selected strong{color:currentColor}.table th{color:#363636}.table th:not([align]){text-align:left}.table tr.is-selected{background-color:#00d1b2;color:#fff}.table tr.is-selected a,.table tr.is-selected strong{color:currentColor}.table tr.is-selected td,.table tr.is-selected th{border-color:#fff;color:currentColor}.table thead{background-color:transparent}.table thead td,.table thead th{border-width:0 0 2px;color:#363636}.table tfoot{background-color:transparent}.table tfoot td,.table tfoot th{border-width:2px 0 0;color:#363636}.table tbody{background-color:transparent}.table tbody tr:last-child td,.table tbody tr:last-child th{border-bottom-width:0}.table.is-bordered td,.table.is-bordered th{border-width:1px}.table.is-bordered tr:last-child td,.table.is-bordered tr:last-child th{border-bottom-width:1px}.table.is-fullwidth{width:100%}.table.is-hoverable tbody tr:not(.is-selected):hover{background-color:#fafafa}.table.is-hoverable.is-striped tbody tr:not(.is-selected):hover{background-color:#fafafa}.table.is-hoverable.is-striped tbody tr:not(.is-selected):hover:nth-child(even){background-color:#f5f5f5}.table.is-narrow td,.table.is-narrow th{padding:.25em .5em}.table.is-striped tbody tr:not(.is-selected):nth-child(even){background-color:#fafafa}.table-container{-webkit-overflow-scrolling:touch;overflow:auto;overflow-y:hidden;max-width:100%}.tags{align-items:center;display:flex;flex-wrap:wrap;justify-content:flex-start}.tags .tag{margin-bottom:.5rem}.tags .tag:not(:last-child){margin-right:.5rem}.tags:last-child{margin-bottom:-.5rem}.tags:not(:last-child){margin-bottom:1rem}.tags.are-medium .tag:not(.is-normal):not(.is-large){font-size:1rem}.tags.are-large .tag:not(.is-normal):not(.is-medium){font-size:1.25rem}.tags.is-centered{justify-content:center}.tags.is-centered .tag{margin-right:.25rem;margin-left:.25rem}.tags.is-right{justify-content:flex-end}.tags.is-right .tag:not(:first-child){margin-left:.5rem}.tags.is-right .tag:not(:last-child){margin-right:0}.tags.has-addons .tag{margin-right:0}.tags.has-addons .tag:not(:first-child){margin-left:0;border-bottom-left-radius:0;border-top-left-radius:0}.tags.has-addons .tag:not(:last-child){border-bottom-right-radius:0;border-top-right-radius:0}.tag:not(body){align-items:center;background-color:#f5f5f5;border-radius:4px;color:#4a4a4a;display:inline-flex;font-size:.75rem;height:2em;justify-content:center;line-height:1.5;padding-left:.75em;padding-right:.75em;white-space:nowrap}.tag:not(body) .delete{margin-left:.25rem;margin-right:-.375rem}.tag:not(body).is-white{background-color:#fff;color:#0a0a0a}.tag:not(body).is-black{background-color:#0a0a0a;color:#fff}.tag:not(body).is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.tag:not(body).is-dark{background-color:#363636;color:#fff}.tag:not(body).is-primary{background-color:#00d1b2;color:#fff}.tag:not(body).is-primary.is-light{background-color:#ebfffc;color:#00947e}.tag:not(body).is-link{background-color:#3273dc;color:#fff}.tag:not(body).is-link.is-light{background-color:#eef3fc;color:#2160c4}.tag:not(body).is-info{background-color:#3298dc;color:#fff}.tag:not(body).is-info.is-light{background-color:#eef6fc;color:#1d72aa}.tag:not(body).is-success{background-color:#48c774;color:#fff}.tag:not(body).is-success.is-light{background-color:#effaf3;color:#257942}.tag:not(body).is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.tag:not(body).is-warning.is-light{background-color:#fffbeb;color:#947600}.tag:not(body).is-danger{background-color:#f14668;color:#fff}.tag:not(body).is-danger.is-light{background-color:#feecf0;color:#cc0f35}.tag:not(body).is-normal{font-size:.75rem}.tag:not(body).is-medium{font-size:1rem}.tag:not(body).is-large{font-size:1.25rem}.tag:not(body) .icon:first-child:not(:last-child){margin-left:-.375em;margin-right:.1875em}.tag:not(body) .icon:last-child:not(:first-child){margin-left:.1875em;margin-right:-.375em}.tag:not(body) .icon:first-child:last-child{margin-left:-.375em;margin-right:-.375em}.tag:not(body).is-delete{margin-left:1px;padding:0;position:relative;width:2em}.tag:not(body).is-delete::after,.tag:not(body).is-delete::before{background-color:currentColor;content:"";display:block;left:50%;position:absolute;top:50%;transform:translateX(-50%) translateY(-50%) rotate(45deg);transform-origin:center center}.tag:not(body).is-delete::before{height:1px;width:50%}.tag:not(body).is-delete::after{height:50%;width:1px}.tag:not(body).is-delete:focus,.tag:not(body).is-delete:hover{background-color:#e8e8e8}.tag:not(body).is-delete:active{background-color:#dbdbdb}.tag:not(body).is-rounded{border-radius:290486px}a.tag:hover{text-decoration:underline}.subtitle,.title{word-break:break-word}.subtitle em,.subtitle span,.title em,.title span{font-weight:inherit}.subtitle sub,.title sub{font-size:.75em}.subtitle sup,.title sup{font-size:.75em}.subtitle .tag,.title .tag{vertical-align:middle}.title{color:#363636;font-size:2rem;font-weight:600;line-height:1.125}.title strong{color:inherit;font-weight:inherit}.title+.highlight{margin-top:-.75rem}.title:not(.is-spaced)+.subtitle{margin-top:-1.25rem}.title.is-1{font-size:3rem}.title.is-2{font-size:2.5rem}.title.is-3{font-size:2rem}.title.is-4{font-size:1.5rem}.title.is-5{font-size:1.25rem}.title.is-6{font-size:1rem}.title.is-7{font-size:.75rem}.subtitle{color:#4a4a4a;font-size:1.25rem;font-weight:400;line-height:1.25}.subtitle strong{color:#363636;font-weight:600}.subtitle:not(.is-spaced)+.title{margin-top:-1.25rem}.subtitle.is-1{font-size:3rem}.subtitle.is-2{font-size:2.5rem}.subtitle.is-3{font-size:2rem}.subtitle.is-4{font-size:1.5rem}.subtitle.is-5{font-size:1.25rem}.subtitle.is-6{font-size:1rem}.subtitle.is-7{font-size:.75rem}.heading{display:block;font-size:11px;letter-spacing:1px;margin-bottom:5px;text-transform:uppercase}.highlight{font-weight:400;max-width:100%;overflow:hidden;padding:0}.highlight pre{overflow:auto;max-width:100%}.number{align-items:center;background-color:#f5f5f5;border-radius:290486px;display:inline-flex;font-size:1.25rem;height:2em;justify-content:center;margin-right:1.5rem;min-width:2.5em;padding:.25rem .5rem;text-align:center;vertical-align:top}.input,.select select,.textarea{background-color:#fff;border-color:#dbdbdb;border-radius:4px;color:#363636}.input::-moz-placeholder,.select select::-moz-placeholder,.textarea::-moz-placeholder{color:rgba(54,54,54,.3)}.input::-webkit-input-placeholder,.select select::-webkit-input-placeholder,.textarea::-webkit-input-placeholder{color:rgba(54,54,54,.3)}.input:-moz-placeholder,.select select:-moz-placeholder,.textarea:-moz-placeholder{color:rgba(54,54,54,.3)}.input:-ms-input-placeholder,.select select:-ms-input-placeholder,.textarea:-ms-input-placeholder{color:rgba(54,54,54,.3)}.input:hover,.is-hovered.input,.is-hovered.textarea,.select select.is-hovered,.select select:hover,.textarea:hover{border-color:#b5b5b5}.input:active,.input:focus,.is-active.input,.is-active.textarea,.is-focused.input,.is-focused.textarea,.select select.is-active,.select select.is-focused,.select select:active,.select select:focus,.textarea:active,.textarea:focus{border-color:#3273dc;box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.input[disabled],.select fieldset[disabled] select,.select select[disabled],.textarea[disabled],fieldset[disabled] .input,fieldset[disabled] .select select,fieldset[disabled] .textarea{background-color:#f5f5f5;border-color:#f5f5f5;box-shadow:none;color:#7a7a7a}.input[disabled]::-moz-placeholder,.select fieldset[disabled] select::-moz-placeholder,.select select[disabled]::-moz-placeholder,.textarea[disabled]::-moz-placeholder,fieldset[disabled] .input::-moz-placeholder,fieldset[disabled] .select select::-moz-placeholder,fieldset[disabled] .textarea::-moz-placeholder{color:rgba(122,122,122,.3)}.input[disabled]::-webkit-input-placeholder,.select fieldset[disabled] select::-webkit-input-placeholder,.select select[disabled]::-webkit-input-placeholder,.textarea[disabled]::-webkit-input-placeholder,fieldset[disabled] .input::-webkit-input-placeholder,fieldset[disabled] .select select::-webkit-input-placeholder,fieldset[disabled] .textarea::-webkit-input-placeholder{color:rgba(122,122,122,.3)}.input[disabled]:-moz-placeholder,.select fieldset[disabled] select:-moz-placeholder,.select select[disabled]:-moz-placeholder,.textarea[disabled]:-moz-placeholder,fieldset[disabled] .input:-moz-placeholder,fieldset[disabled] .select select:-moz-placeholder,fieldset[disabled] .textarea:-moz-placeholder{color:rgba(122,122,122,.3)}.input[disabled]:-ms-input-placeholder,.select fieldset[disabled] select:-ms-input-placeholder,.select select[disabled]:-ms-input-placeholder,.textarea[disabled]:-ms-input-placeholder,fieldset[disabled] .input:-ms-input-placeholder,fieldset[disabled] .select select:-ms-input-placeholder,fieldset[disabled] .textarea:-ms-input-placeholder{color:rgba(122,122,122,.3)}.input,.textarea{box-shadow:inset 0 .0625em .125em rgba(10,10,10,.05);max-width:100%;width:100%}.input[readonly],.textarea[readonly]{box-shadow:none}.is-white.input,.is-white.textarea{border-color:#fff}.is-white.input:active,.is-white.input:focus,.is-white.is-active.input,.is-white.is-active.textarea,.is-white.is-focused.input,.is-white.is-focused.textarea,.is-white.textarea:active,.is-white.textarea:focus{box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.is-black.input,.is-black.textarea{border-color:#0a0a0a}.is-black.input:active,.is-black.input:focus,.is-black.is-active.input,.is-black.is-active.textarea,.is-black.is-focused.input,.is-black.is-focused.textarea,.is-black.textarea:active,.is-black.textarea:focus{box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.is-light.input,.is-light.textarea{border-color:#f5f5f5}.is-light.input:active,.is-light.input:focus,.is-light.is-active.input,.is-light.is-active.textarea,.is-light.is-focused.input,.is-light.is-focused.textarea,.is-light.textarea:active,.is-light.textarea:focus{box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.is-dark.input,.is-dark.textarea{border-color:#363636}.is-dark.input:active,.is-dark.input:focus,.is-dark.is-active.input,.is-dark.is-active.textarea,.is-dark.is-focused.input,.is-dark.is-focused.textarea,.is-dark.textarea:active,.is-dark.textarea:focus{box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.is-primary.input,.is-primary.textarea{border-color:#00d1b2}.is-primary.input:active,.is-primary.input:focus,.is-primary.is-active.input,.is-primary.is-active.textarea,.is-primary.is-focused.input,.is-primary.is-focused.textarea,.is-primary.textarea:active,.is-primary.textarea:focus{box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.is-link.input,.is-link.textarea{border-color:#3273dc}.is-link.input:active,.is-link.input:focus,.is-link.is-active.input,.is-link.is-active.textarea,.is-link.is-focused.input,.is-link.is-focused.textarea,.is-link.textarea:active,.is-link.textarea:focus{box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.is-info.input,.is-info.textarea{border-color:#3298dc}.is-info.input:active,.is-info.input:focus,.is-info.is-active.input,.is-info.is-active.textarea,.is-info.is-focused.input,.is-info.is-focused.textarea,.is-info.textarea:active,.is-info.textarea:focus{box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.is-success.input,.is-success.textarea{border-color:#48c774}.is-success.input:active,.is-success.input:focus,.is-success.is-active.input,.is-success.is-active.textarea,.is-success.is-focused.input,.is-success.is-focused.textarea,.is-success.textarea:active,.is-success.textarea:focus{box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.is-warning.input,.is-warning.textarea{border-color:#ffdd57}.is-warning.input:active,.is-warning.input:focus,.is-warning.is-active.input,.is-warning.is-active.textarea,.is-warning.is-focused.input,.is-warning.is-focused.textarea,.is-warning.textarea:active,.is-warning.textarea:focus{box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.is-danger.input,.is-danger.textarea{border-color:#f14668}.is-danger.input:active,.is-danger.input:focus,.is-danger.is-active.input,.is-danger.is-active.textarea,.is-danger.is-focused.input,.is-danger.is-focused.textarea,.is-danger.textarea:active,.is-danger.textarea:focus{box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.is-small.input,.is-small.textarea{border-radius:2px;font-size:.75rem}.is-medium.input,.is-medium.textarea{font-size:1.25rem}.is-large.input,.is-large.textarea{font-size:1.5rem}.is-fullwidth.input,.is-fullwidth.textarea{display:block;width:100%}.is-inline.input,.is-inline.textarea{display:inline;width:auto}.input.is-rounded{border-radius:290486px;padding-left:calc(calc(.75em - 1px) + .375em);padding-right:calc(calc(.75em - 1px) + .375em)}.input.is-static{background-color:transparent;border-color:transparent;box-shadow:none;padding-left:0;padding-right:0}.textarea{display:block;max-width:100%;min-width:100%;padding:calc(.75em - 1px);resize:vertical}.textarea:not([rows]){max-height:40em;min-height:8em}.textarea[rows]{height:initial}.textarea.has-fixed-size{resize:none}.checkbox,.radio{cursor:pointer;display:inline-block;line-height:1.25;position:relative}.checkbox input,.radio input{cursor:pointer}.checkbox:hover,.radio:hover{color:#363636}.checkbox[disabled],.radio[disabled],fieldset[disabled] .checkbox,fieldset[disabled] .radio{color:#7a7a7a;cursor:not-allowed}.radio+.radio{margin-left:.5em}.select{display:inline-block;max-width:100%;position:relative;vertical-align:top}.select:not(.is-multiple){height:2.5em}.select:not(.is-multiple):not(.is-loading)::after{border-color:#3273dc;right:1.125em;z-index:4}.select.is-rounded select{border-radius:290486px;padding-left:1em}.select select{cursor:pointer;display:block;font-size:1em;max-width:100%;outline:0}.select select::-ms-expand{display:none}.select select[disabled]:hover,fieldset[disabled] .select select:hover{border-color:#f5f5f5}.select select:not([multiple]){padding-right:2.5em}.select select[multiple]{height:auto;padding:0}.select select[multiple] option{padding:.5em 1em}.select:not(.is-multiple):not(.is-loading):hover::after{border-color:#363636}.select.is-white:not(:hover)::after{border-color:#fff}.select.is-white select{border-color:#fff}.select.is-white select.is-hovered,.select.is-white select:hover{border-color:#f2f2f2}.select.is-white select.is-active,.select.is-white select.is-focused,.select.is-white select:active,.select.is-white select:focus{box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.select.is-black:not(:hover)::after{border-color:#0a0a0a}.select.is-black select{border-color:#0a0a0a}.select.is-black select.is-hovered,.select.is-black select:hover{border-color:#000}.select.is-black select.is-active,.select.is-black select.is-focused,.select.is-black select:active,.select.is-black select:focus{box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.select.is-light:not(:hover)::after{border-color:#f5f5f5}.select.is-light select{border-color:#f5f5f5}.select.is-light select.is-hovered,.select.is-light select:hover{border-color:#e8e8e8}.select.is-light select.is-active,.select.is-light select.is-focused,.select.is-light select:active,.select.is-light select:focus{box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.select.is-dark:not(:hover)::after{border-color:#363636}.select.is-dark select{border-color:#363636}.select.is-dark select.is-hovered,.select.is-dark select:hover{border-color:#292929}.select.is-dark select.is-active,.select.is-dark select.is-focused,.select.is-dark select:active,.select.is-dark select:focus{box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.select.is-primary:not(:hover)::after{border-color:#00d1b2}.select.is-primary select{border-color:#00d1b2}.select.is-primary select.is-hovered,.select.is-primary select:hover{border-color:#00b89c}.select.is-primary select.is-active,.select.is-primary select.is-focused,.select.is-primary select:active,.select.is-primary select:focus{box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.select.is-link:not(:hover)::after{border-color:#3273dc}.select.is-link select{border-color:#3273dc}.select.is-link select.is-hovered,.select.is-link select:hover{border-color:#2366d1}.select.is-link select.is-active,.select.is-link select.is-focused,.select.is-link select:active,.select.is-link select:focus{box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.select.is-info:not(:hover)::after{border-color:#3298dc}.select.is-info select{border-color:#3298dc}.select.is-info select.is-hovered,.select.is-info select:hover{border-color:#238cd1}.select.is-info select.is-active,.select.is-info select.is-focused,.select.is-info select:active,.select.is-info select:focus{box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.select.is-success:not(:hover)::after{border-color:#48c774}.select.is-success select{border-color:#48c774}.select.is-success select.is-hovered,.select.is-success select:hover{border-color:#3abb67}.select.is-success select.is-active,.select.is-success select.is-focused,.select.is-success select:active,.select.is-success select:focus{box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.select.is-warning:not(:hover)::after{border-color:#ffdd57}.select.is-warning select{border-color:#ffdd57}.select.is-warning select.is-hovered,.select.is-warning select:hover{border-color:#ffd83d}.select.is-warning select.is-active,.select.is-warning select.is-focused,.select.is-warning select:active,.select.is-warning select:focus{box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.select.is-danger:not(:hover)::after{border-color:#f14668}.select.is-danger select{border-color:#f14668}.select.is-danger select.is-hovered,.select.is-danger select:hover{border-color:#ef2e55}.select.is-danger select.is-active,.select.is-danger select.is-focused,.select.is-danger select:active,.select.is-danger select:focus{box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.select.is-small{border-radius:2px;font-size:.75rem}.select.is-medium{font-size:1.25rem}.select.is-large{font-size:1.5rem}.select.is-disabled::after{border-color:#7a7a7a}.select.is-fullwidth{width:100%}.select.is-fullwidth select{width:100%}.select.is-loading::after{margin-top:0;position:absolute;right:.625em;top:.625em;transform:none}.select.is-loading.is-small:after{font-size:.75rem}.select.is-loading.is-medium:after{font-size:1.25rem}.select.is-loading.is-large:after{font-size:1.5rem}.file{align-items:stretch;display:flex;justify-content:flex-start;position:relative}.file.is-white .file-cta{background-color:#fff;border-color:transparent;color:#0a0a0a}.file.is-white.is-hovered .file-cta,.file.is-white:hover .file-cta{background-color:#f9f9f9;border-color:transparent;color:#0a0a0a}.file.is-white.is-focused .file-cta,.file.is-white:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(255,255,255,.25);color:#0a0a0a}.file.is-white.is-active .file-cta,.file.is-white:active .file-cta{background-color:#f2f2f2;border-color:transparent;color:#0a0a0a}.file.is-black .file-cta{background-color:#0a0a0a;border-color:transparent;color:#fff}.file.is-black.is-hovered .file-cta,.file.is-black:hover .file-cta{background-color:#040404;border-color:transparent;color:#fff}.file.is-black.is-focused .file-cta,.file.is-black:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(10,10,10,.25);color:#fff}.file.is-black.is-active .file-cta,.file.is-black:active .file-cta{background-color:#000;border-color:transparent;color:#fff}.file.is-light .file-cta{background-color:#f5f5f5;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-light.is-hovered .file-cta,.file.is-light:hover .file-cta{background-color:#eee;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-light.is-focused .file-cta,.file.is-light:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(245,245,245,.25);color:rgba(0,0,0,.7)}.file.is-light.is-active .file-cta,.file.is-light:active .file-cta{background-color:#e8e8e8;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-dark .file-cta{background-color:#363636;border-color:transparent;color:#fff}.file.is-dark.is-hovered .file-cta,.file.is-dark:hover .file-cta{background-color:#2f2f2f;border-color:transparent;color:#fff}.file.is-dark.is-focused .file-cta,.file.is-dark:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(54,54,54,.25);color:#fff}.file.is-dark.is-active .file-cta,.file.is-dark:active .file-cta{background-color:#292929;border-color:transparent;color:#fff}.file.is-primary .file-cta{background-color:#00d1b2;border-color:transparent;color:#fff}.file.is-primary.is-hovered .file-cta,.file.is-primary:hover .file-cta{background-color:#00c4a7;border-color:transparent;color:#fff}.file.is-primary.is-focused .file-cta,.file.is-primary:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(0,209,178,.25);color:#fff}.file.is-primary.is-active .file-cta,.file.is-primary:active .file-cta{background-color:#00b89c;border-color:transparent;color:#fff}.file.is-link .file-cta{background-color:#3273dc;border-color:transparent;color:#fff}.file.is-link.is-hovered .file-cta,.file.is-link:hover .file-cta{background-color:#276cda;border-color:transparent;color:#fff}.file.is-link.is-focused .file-cta,.file.is-link:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(50,115,220,.25);color:#fff}.file.is-link.is-active .file-cta,.file.is-link:active .file-cta{background-color:#2366d1;border-color:transparent;color:#fff}.file.is-info .file-cta{background-color:#3298dc;border-color:transparent;color:#fff}.file.is-info.is-hovered .file-cta,.file.is-info:hover .file-cta{background-color:#2793da;border-color:transparent;color:#fff}.file.is-info.is-focused .file-cta,.file.is-info:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(50,152,220,.25);color:#fff}.file.is-info.is-active .file-cta,.file.is-info:active .file-cta{background-color:#238cd1;border-color:transparent;color:#fff}.file.is-success .file-cta{background-color:#48c774;border-color:transparent;color:#fff}.file.is-success.is-hovered .file-cta,.file.is-success:hover .file-cta{background-color:#3ec46d;border-color:transparent;color:#fff}.file.is-success.is-focused .file-cta,.file.is-success:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(72,199,116,.25);color:#fff}.file.is-success.is-active .file-cta,.file.is-success:active .file-cta{background-color:#3abb67;border-color:transparent;color:#fff}.file.is-warning .file-cta{background-color:#ffdd57;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-warning.is-hovered .file-cta,.file.is-warning:hover .file-cta{background-color:#ffdb4a;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-warning.is-focused .file-cta,.file.is-warning:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(255,221,87,.25);color:rgba(0,0,0,.7)}.file.is-warning.is-active .file-cta,.file.is-warning:active .file-cta{background-color:#ffd83d;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-danger .file-cta{background-color:#f14668;border-color:transparent;color:#fff}.file.is-danger.is-hovered .file-cta,.file.is-danger:hover .file-cta{background-color:#f03a5f;border-color:transparent;color:#fff}.file.is-danger.is-focused .file-cta,.file.is-danger:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(241,70,104,.25);color:#fff}.file.is-danger.is-active .file-cta,.file.is-danger:active .file-cta{background-color:#ef2e55;border-color:transparent;color:#fff}.file.is-small{font-size:.75rem}.file.is-medium{font-size:1.25rem}.file.is-medium .file-icon .fa{font-size:21px}.file.is-large{font-size:1.5rem}.file.is-large .file-icon .fa{font-size:28px}.file.has-name .file-cta{border-bottom-right-radius:0;border-top-right-radius:0}.file.has-name .file-name{border-bottom-left-radius:0;border-top-left-radius:0}.file.has-name.is-empty .file-cta{border-radius:4px}.file.has-name.is-empty .file-name{display:none}.file.is-boxed .file-label{flex-direction:column}.file.is-boxed .file-cta{flex-direction:column;height:auto;padding:1em 3em}.file.is-boxed .file-name{border-width:0 1px 1px}.file.is-boxed .file-icon{height:1.5em;width:1.5em}.file.is-boxed .file-icon .fa{font-size:21px}.file.is-boxed.is-small .file-icon .fa{font-size:14px}.file.is-boxed.is-medium .file-icon .fa{font-size:28px}.file.is-boxed.is-large .file-icon .fa{font-size:35px}.file.is-boxed.has-name .file-cta{border-radius:4px 4px 0 0}.file.is-boxed.has-name .file-name{border-radius:0 0 4px 4px;border-width:0 1px 1px}.file.is-centered{justify-content:center}.file.is-fullwidth .file-label{width:100%}.file.is-fullwidth .file-name{flex-grow:1;max-width:none}.file.is-right{justify-content:flex-end}.file.is-right .file-cta{border-radius:0 4px 4px 0}.file.is-right .file-name{border-radius:4px 0 0 4px;border-width:1px 0 1px 1px;order:-1}.file-label{align-items:stretch;display:flex;cursor:pointer;justify-content:flex-start;overflow:hidden;position:relative}.file-label:hover .file-cta{background-color:#eee;color:#363636}.file-label:hover .file-name{border-color:#d5d5d5}.file-label:active .file-cta{background-color:#e8e8e8;color:#363636}.file-label:active .file-name{border-color:#cfcfcf}.file-input{height:100%;left:0;opacity:0;outline:0;position:absolute;top:0;width:100%}.file-cta,.file-name{border-color:#dbdbdb;border-radius:4px;font-size:1em;padding-left:1em;padding-right:1em;white-space:nowrap}.file-cta{background-color:#f5f5f5;color:#4a4a4a}.file-name{border-color:#dbdbdb;border-style:solid;border-width:1px 1px 1px 0;display:block;max-width:16em;overflow:hidden;text-align:left;text-overflow:ellipsis}.file-icon{align-items:center;display:flex;height:1em;justify-content:center;margin-right:.5em;width:1em}.file-icon .fa{font-size:14px}.label{color:#363636;display:block;font-size:1rem;font-weight:700}.label:not(:last-child){margin-bottom:.5em}.label.is-small{font-size:.75rem}.label.is-medium{font-size:1.25rem}.label.is-large{font-size:1.5rem}.help{display:block;font-size:.75rem;margin-top:.25rem}.help.is-white{color:#fff}.help.is-black{color:#0a0a0a}.help.is-light{color:#f5f5f5}.help.is-dark{color:#363636}.help.is-primary{color:#00d1b2}.help.is-link{color:#3273dc}.help.is-info{color:#3298dc}.help.is-success{color:#48c774}.help.is-warning{color:#ffdd57}.help.is-danger{color:#f14668}.field:not(:last-child){margin-bottom:.75rem}.field.has-addons{display:flex;justify-content:flex-start}.field.has-addons .control:not(:last-child){margin-right:-1px}.field.has-addons .control:not(:first-child):not(:last-child) .button,.field.has-addons .control:not(:first-child):not(:last-child) .input,.field.has-addons .control:not(:first-child):not(:last-child) .select select{border-radius:0}.field.has-addons .control:first-child:not(:only-child) .button,.field.has-addons .control:first-child:not(:only-child) .input,.field.has-addons .control:first-child:not(:only-child) .select select{border-bottom-right-radius:0;border-top-right-radius:0}.field.has-addons .control:last-child:not(:only-child) .button,.field.has-addons .control:last-child:not(:only-child) .input,.field.has-addons .control:last-child:not(:only-child) .select select{border-bottom-left-radius:0;border-top-left-radius:0}.field.has-addons .control .button:not([disabled]).is-hovered,.field.has-addons .control .button:not([disabled]):hover,.field.has-addons .control .input:not([disabled]).is-hovered,.field.has-addons .control .input:not([disabled]):hover,.field.has-addons .control .select select:not([disabled]).is-hovered,.field.has-addons .control .select select:not([disabled]):hover{z-index:2}.field.has-addons .control .button:not([disabled]).is-active,.field.has-addons .control .button:not([disabled]).is-focused,.field.has-addons .control .button:not([disabled]):active,.field.has-addons .control .button:not([disabled]):focus,.field.has-addons .control .input:not([disabled]).is-active,.field.has-addons .control .input:not([disabled]).is-focused,.field.has-addons .control .input:not([disabled]):active,.field.has-addons .control .input:not([disabled]):focus,.field.has-addons .control .select select:not([disabled]).is-active,.field.has-addons .control .select select:not([disabled]).is-focused,.field.has-addons .control .select select:not([disabled]):active,.field.has-addons .control .select select:not([disabled]):focus{z-index:3}.field.has-addons .control .button:not([disabled]).is-active:hover,.field.has-addons .control .button:not([disabled]).is-focused:hover,.field.has-addons .control .button:not([disabled]):active:hover,.field.has-addons .control .button:not([disabled]):focus:hover,.field.has-addons .control .input:not([disabled]).is-active:hover,.field.has-addons .control .input:not([disabled]).is-focused:hover,.field.has-addons .control .input:not([disabled]):active:hover,.field.has-addons .control .input:not([disabled]):focus:hover,.field.has-addons .control .select select:not([disabled]).is-active:hover,.field.has-addons .control .select select:not([disabled]).is-focused:hover,.field.has-addons .control .select select:not([disabled]):active:hover,.field.has-addons .control .select select:not([disabled]):focus:hover{z-index:4}.field.has-addons .control.is-expanded{flex-grow:1;flex-shrink:1}.field.has-addons.has-addons-centered{justify-content:center}.field.has-addons.has-addons-right{justify-content:flex-end}.field.has-addons.has-addons-fullwidth .control{flex-grow:1;flex-shrink:0}.field.is-grouped{display:flex;justify-content:flex-start}.field.is-grouped>.control{flex-shrink:0}.field.is-grouped>.control:not(:last-child){margin-bottom:0;margin-right:.75rem}.field.is-grouped>.control.is-expanded{flex-grow:1;flex-shrink:1}.field.is-grouped.is-grouped-centered{justify-content:center}.field.is-grouped.is-grouped-right{justify-content:flex-end}.field.is-grouped.is-grouped-multiline{flex-wrap:wrap}.field.is-grouped.is-grouped-multiline>.control:last-child,.field.is-grouped.is-grouped-multiline>.control:not(:last-child){margin-bottom:.75rem}.field.is-grouped.is-grouped-multiline:last-child{margin-bottom:-.75rem}.field.is-grouped.is-grouped-multiline:not(:last-child){margin-bottom:0}@media screen and (min-width:769px),print{.field.is-horizontal{display:flex}}.field-label .label{font-size:inherit}@media screen and (max-width:768px){.field-label{margin-bottom:.5rem}}@media screen and (min-width:769px),print{.field-label{flex-basis:0;flex-grow:1;flex-shrink:0;margin-right:1.5rem;text-align:right}.field-label.is-small{font-size:.75rem;padding-top:.375em}.field-label.is-normal{padding-top:.375em}.field-label.is-medium{font-size:1.25rem;padding-top:.375em}.field-label.is-large{font-size:1.5rem;padding-top:.375em}}.field-body .field .field{margin-bottom:0}@media screen and (min-width:769px),print{.field-body{display:flex;flex-basis:0;flex-grow:5;flex-shrink:1}.field-body .field{margin-bottom:0}.field-body>.field{flex-shrink:1}.field-body>.field:not(.is-narrow){flex-grow:1}.field-body>.field:not(:last-child){margin-right:.75rem}}.control{box-sizing:border-box;clear:both;font-size:1rem;position:relative;text-align:left}.control.has-icons-left .input:focus~.icon,.control.has-icons-left .select:focus~.icon,.control.has-icons-right .input:focus~.icon,.control.has-icons-right .select:focus~.icon{color:#4a4a4a}.control.has-icons-left .input.is-small~.icon,.control.has-icons-left .select.is-small~.icon,.control.has-icons-right .input.is-small~.icon,.control.has-icons-right .select.is-small~.icon{font-size:.75rem}.control.has-icons-left .input.is-medium~.icon,.control.has-icons-left .select.is-medium~.icon,.control.has-icons-right .input.is-medium~.icon,.control.has-icons-right .select.is-medium~.icon{font-size:1.25rem}.control.has-icons-left .input.is-large~.icon,.control.has-icons-left .select.is-large~.icon,.control.has-icons-right .input.is-large~.icon,.control.has-icons-right .select.is-large~.icon{font-size:1.5rem}.control.has-icons-left .icon,.control.has-icons-right .icon{color:#dbdbdb;height:2.5em;pointer-events:none;position:absolute;top:0;width:2.5em;z-index:4}.control.has-icons-left .input,.control.has-icons-left .select select{padding-left:2.5em}.control.has-icons-left .icon.is-left{left:0}.control.has-icons-right .input,.control.has-icons-right .select select{padding-right:2.5em}.control.has-icons-right .icon.is-right{right:0}.control.is-loading::after{position:absolute!important;right:.625em;top:.625em;z-index:4}.control.is-loading.is-small:after{font-size:.75rem}.control.is-loading.is-medium:after{font-size:1.25rem}.control.is-loading.is-large:after{font-size:1.5rem}.breadcrumb{font-size:1rem;white-space:nowrap}.breadcrumb a{align-items:center;color:#3273dc;display:flex;justify-content:center;padding:0 .75em}.breadcrumb a:hover{color:#363636}.breadcrumb li{align-items:center;display:flex}.breadcrumb li:first-child a{padding-left:0}.breadcrumb li.is-active a{color:#363636;cursor:default;pointer-events:none}.breadcrumb li+li::before{color:#b5b5b5;content:"\0002f"}.breadcrumb ol,.breadcrumb ul{align-items:flex-start;display:flex;flex-wrap:wrap;justify-content:flex-start}.breadcrumb .icon:first-child{margin-right:.5em}.breadcrumb .icon:last-child{margin-left:.5em}.breadcrumb.is-centered ol,.breadcrumb.is-centered ul{justify-content:center}.breadcrumb.is-right ol,.breadcrumb.is-right ul{justify-content:flex-end}.breadcrumb.is-small{font-size:.75rem}.breadcrumb.is-medium{font-size:1.25rem}.breadcrumb.is-large{font-size:1.5rem}.breadcrumb.has-arrow-separator li+li::before{content:"\02192"}.breadcrumb.has-bullet-separator li+li::before{content:"\02022"}.breadcrumb.has-dot-separator li+li::before{content:"\000b7"}.breadcrumb.has-succeeds-separator li+li::before{content:"\0227B"}.card{background-color:#fff;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);color:#4a4a4a;max-width:100%;position:relative}.card-header{background-color:transparent;align-items:stretch;box-shadow:0 .125em .25em rgba(10,10,10,.1);display:flex}.card-header-title{align-items:center;color:#363636;display:flex;flex-grow:1;font-weight:700;padding:.75rem 1rem}.card-header-title.is-centered{justify-content:center}.card-header-icon{align-items:center;cursor:pointer;display:flex;justify-content:center;padding:.75rem 1rem}.card-image{display:block;position:relative}.card-content{background-color:transparent;padding:1.5rem}.card-footer{background-color:transparent;border-top:1px solid #ededed;align-items:stretch;display:flex}.card-footer-item{align-items:center;display:flex;flex-basis:0;flex-grow:1;flex-shrink:0;justify-content:center;padding:.75rem}.card-footer-item:not(:last-child){border-right:1px solid #ededed}.card .media:not(:last-child){margin-bottom:1.5rem}.dropdown{display:inline-flex;position:relative;vertical-align:top}.dropdown.is-active .dropdown-menu,.dropdown.is-hoverable:hover .dropdown-menu{display:block}.dropdown.is-right .dropdown-menu{left:auto;right:0}.dropdown.is-up .dropdown-menu{bottom:100%;padding-bottom:4px;padding-top:initial;top:auto}.dropdown-menu{display:none;left:0;min-width:12rem;padding-top:4px;position:absolute;top:100%;z-index:20}.dropdown-content{background-color:#fff;border-radius:4px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);padding-bottom:.5rem;padding-top:.5rem}.dropdown-item{color:#4a4a4a;display:block;font-size:.875rem;line-height:1.5;padding:.375rem 1rem;position:relative}a.dropdown-item,button.dropdown-item{padding-right:3rem;text-align:left;white-space:nowrap;width:100%}a.dropdown-item:hover,button.dropdown-item:hover{background-color:#f5f5f5;color:#0a0a0a}a.dropdown-item.is-active,button.dropdown-item.is-active{background-color:#3273dc;color:#fff}.dropdown-divider{background-color:#ededed;border:none;display:block;height:1px;margin:.5rem 0}.level{align-items:center;justify-content:space-between}.level code{border-radius:4px}.level img{display:inline-block;vertical-align:top}.level.is-mobile{display:flex}.level.is-mobile .level-left,.level.is-mobile .level-right{display:flex}.level.is-mobile .level-left+.level-right{margin-top:0}.level.is-mobile .level-item:not(:last-child){margin-bottom:0;margin-right:.75rem}.level.is-mobile .level-item:not(.is-narrow){flex-grow:1}@media screen and (min-width:769px),print{.level{display:flex}.level>.level-item:not(.is-narrow){flex-grow:1}}.level-item{align-items:center;display:flex;flex-basis:auto;flex-grow:0;flex-shrink:0;justify-content:center}.level-item .subtitle,.level-item .title{margin-bottom:0}@media screen and (max-width:768px){.level-item:not(:last-child){margin-bottom:.75rem}}.level-left,.level-right{flex-basis:auto;flex-grow:0;flex-shrink:0}.level-left .level-item.is-flexible,.level-right .level-item.is-flexible{flex-grow:1}@media screen and (min-width:769px),print{.level-left .level-item:not(:last-child),.level-right .level-item:not(:last-child){margin-right:.75rem}}.level-left{align-items:center;justify-content:flex-start}@media screen and (max-width:768px){.level-left+.level-right{margin-top:1.5rem}}@media screen and (min-width:769px),print{.level-left{display:flex}}.level-right{align-items:center;justify-content:flex-end}@media screen and (min-width:769px),print{.level-right{display:flex}}.list{background-color:#fff;border-radius:4px;box-shadow:0 2px 3px rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.1)}.list-item{display:block;padding:.5em 1em}.list-item:not(a){color:#4a4a4a}.list-item:first-child{border-top-left-radius:4px;border-top-right-radius:4px}.list-item:last-child{border-bottom-left-radius:4px;border-bottom-right-radius:4px}.list-item:not(:last-child){border-bottom:1px solid #dbdbdb}.list-item.is-active{background-color:#3273dc;color:#fff}a.list-item{background-color:#f5f5f5;cursor:pointer}.media{align-items:flex-start;display:flex;text-align:left}.media .content:not(:last-child){margin-bottom:.75rem}.media .media{border-top:1px solid rgba(219,219,219,.5);display:flex;padding-top:.75rem}.media .media .content:not(:last-child),.media .media .control:not(:last-child){margin-bottom:.5rem}.media .media .media{padding-top:.5rem}.media .media .media+.media{margin-top:.5rem}.media+.media{border-top:1px solid rgba(219,219,219,.5);margin-top:1rem;padding-top:1rem}.media.is-large+.media{margin-top:1.5rem;padding-top:1.5rem}.media-left,.media-right{flex-basis:auto;flex-grow:0;flex-shrink:0}.media-left{margin-right:1rem}.media-right{margin-left:1rem}.media-content{flex-basis:auto;flex-grow:1;flex-shrink:1;text-align:left}@media screen and (max-width:768px){.media-content{overflow-x:auto}}.menu{font-size:1rem}.menu.is-small{font-size:.75rem}.menu.is-medium{font-size:1.25rem}.menu.is-large{font-size:1.5rem}.menu-list{line-height:1.25}.menu-list a{border-radius:2px;color:#4a4a4a;display:block;padding:.5em .75em}.menu-list a:hover{background-color:#f5f5f5;color:#363636}.menu-list a.is-active{background-color:#3273dc;color:#fff}.menu-list li ul{border-left:1px solid #dbdbdb;margin:.75em;padding-left:.75em}.menu-label{color:#7a7a7a;font-size:.75em;letter-spacing:.1em;text-transform:uppercase}.menu-label:not(:first-child){margin-top:1em}.menu-label:not(:last-child){margin-bottom:1em}.message{background-color:#f5f5f5;border-radius:4px;font-size:1rem}.message strong{color:currentColor}.message a:not(.button):not(.tag):not(.dropdown-item){color:currentColor;text-decoration:underline}.message.is-small{font-size:.75rem}.message.is-medium{font-size:1.25rem}.message.is-large{font-size:1.5rem}.message.is-white{background-color:#fff}.message.is-white .message-header{background-color:#fff;color:#0a0a0a}.message.is-white .message-body{border-color:#fff}.message.is-black{background-color:#fafafa}.message.is-black .message-header{background-color:#0a0a0a;color:#fff}.message.is-black .message-body{border-color:#0a0a0a}.message.is-light{background-color:#fafafa}.message.is-light .message-header{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.message.is-light .message-body{border-color:#f5f5f5}.message.is-dark{background-color:#fafafa}.message.is-dark .message-header{background-color:#363636;color:#fff}.message.is-dark .message-body{border-color:#363636}.message.is-primary{background-color:#ebfffc}.message.is-primary .message-header{background-color:#00d1b2;color:#fff}.message.is-primary .message-body{border-color:#00d1b2;color:#00947e}.message.is-link{background-color:#eef3fc}.message.is-link .message-header{background-color:#3273dc;color:#fff}.message.is-link .message-body{border-color:#3273dc;color:#2160c4}.message.is-info{background-color:#eef6fc}.message.is-info .message-header{background-color:#3298dc;color:#fff}.message.is-info .message-body{border-color:#3298dc;color:#1d72aa}.message.is-success{background-color:#effaf3}.message.is-success .message-header{background-color:#48c774;color:#fff}.message.is-success .message-body{border-color:#48c774;color:#257942}.message.is-warning{background-color:#fffbeb}.message.is-warning .message-header{background-color:#ffdd57;color:rgba(0,0,0,.7)}.message.is-warning .message-body{border-color:#ffdd57;color:#947600}.message.is-danger{background-color:#feecf0}.message.is-danger .message-header{background-color:#f14668;color:#fff}.message.is-danger .message-body{border-color:#f14668;color:#cc0f35}.message-header{align-items:center;background-color:#4a4a4a;border-radius:4px 4px 0 0;color:#fff;display:flex;font-weight:700;justify-content:space-between;line-height:1.25;padding:.75em 1em;position:relative}.message-header .delete{flex-grow:0;flex-shrink:0;margin-left:.75em}.message-header+.message-body{border-width:0;border-top-left-radius:0;border-top-right-radius:0}.message-body{border-color:#dbdbdb;border-radius:4px;border-style:solid;border-width:0 0 0 4px;color:#4a4a4a;padding:1.25em 1.5em}.message-body code,.message-body pre{background-color:#fff}.message-body pre code{background-color:transparent}.modal{align-items:center;display:none;flex-direction:column;justify-content:center;overflow:hidden;position:fixed;z-index:40}.modal.is-active{display:flex}.modal-background{background-color:rgba(10,10,10,.86)}.modal-card,.modal-content{margin:0 20px;max-height:calc(100vh - 160px);overflow:auto;position:relative;width:100%}@media screen and (min-width:769px),print{.modal-card,.modal-content{margin:0 auto;max-height:calc(100vh - 40px);width:640px}}.modal-close{background:0 0;height:40px;position:fixed;right:20px;top:20px;width:40px}.modal-card{display:flex;flex-direction:column;max-height:calc(100vh - 40px);overflow:hidden;-ms-overflow-y:visible}.modal-card-foot,.modal-card-head{align-items:center;background-color:#f5f5f5;display:flex;flex-shrink:0;justify-content:flex-start;padding:20px;position:relative}.modal-card-head{border-bottom:1px solid #dbdbdb;border-top-left-radius:6px;border-top-right-radius:6px}.modal-card-title{color:#363636;flex-grow:1;flex-shrink:0;font-size:1.5rem;line-height:1}.modal-card-foot{border-bottom-left-radius:6px;border-bottom-right-radius:6px;border-top:1px solid #dbdbdb}.modal-card-foot .button:not(:last-child){margin-right:.5em}.modal-card-body{-webkit-overflow-scrolling:touch;background-color:#fff;flex-grow:1;flex-shrink:1;overflow:auto;padding:20px}.navbar{background-color:#fff;min-height:3.25rem;position:relative;z-index:30}.navbar.is-white{background-color:#fff;color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link,.navbar.is-white .navbar-brand>.navbar-item{color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link.is-active,.navbar.is-white .navbar-brand .navbar-link:focus,.navbar.is-white .navbar-brand .navbar-link:hover,.navbar.is-white .navbar-brand>a.navbar-item.is-active,.navbar.is-white .navbar-brand>a.navbar-item:focus,.navbar.is-white .navbar-brand>a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link::after{border-color:#0a0a0a}.navbar.is-white .navbar-burger{color:#0a0a0a}@media screen and (min-width:1024px){.navbar.is-white .navbar-end .navbar-link,.navbar.is-white .navbar-end>.navbar-item,.navbar.is-white .navbar-start .navbar-link,.navbar.is-white .navbar-start>.navbar-item{color:#0a0a0a}.navbar.is-white .navbar-end .navbar-link.is-active,.navbar.is-white .navbar-end .navbar-link:focus,.navbar.is-white .navbar-end .navbar-link:hover,.navbar.is-white .navbar-end>a.navbar-item.is-active,.navbar.is-white .navbar-end>a.navbar-item:focus,.navbar.is-white .navbar-end>a.navbar-item:hover,.navbar.is-white .navbar-start .navbar-link.is-active,.navbar.is-white .navbar-start .navbar-link:focus,.navbar.is-white .navbar-start .navbar-link:hover,.navbar.is-white .navbar-start>a.navbar-item.is-active,.navbar.is-white .navbar-start>a.navbar-item:focus,.navbar.is-white .navbar-start>a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-end .navbar-link::after,.navbar.is-white .navbar-start .navbar-link::after{border-color:#0a0a0a}.navbar.is-white .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-white .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-white .navbar-item.has-dropdown:hover .navbar-link{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-dropdown a.navbar-item.is-active{background-color:#fff;color:#0a0a0a}}.navbar.is-black{background-color:#0a0a0a;color:#fff}.navbar.is-black .navbar-brand .navbar-link,.navbar.is-black .navbar-brand>.navbar-item{color:#fff}.navbar.is-black .navbar-brand .navbar-link.is-active,.navbar.is-black .navbar-brand .navbar-link:focus,.navbar.is-black .navbar-brand .navbar-link:hover,.navbar.is-black .navbar-brand>a.navbar-item.is-active,.navbar.is-black .navbar-brand>a.navbar-item:focus,.navbar.is-black .navbar-brand>a.navbar-item:hover{background-color:#000;color:#fff}.navbar.is-black .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-black .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-black .navbar-end .navbar-link,.navbar.is-black .navbar-end>.navbar-item,.navbar.is-black .navbar-start .navbar-link,.navbar.is-black .navbar-start>.navbar-item{color:#fff}.navbar.is-black .navbar-end .navbar-link.is-active,.navbar.is-black .navbar-end .navbar-link:focus,.navbar.is-black .navbar-end .navbar-link:hover,.navbar.is-black .navbar-end>a.navbar-item.is-active,.navbar.is-black .navbar-end>a.navbar-item:focus,.navbar.is-black .navbar-end>a.navbar-item:hover,.navbar.is-black .navbar-start .navbar-link.is-active,.navbar.is-black .navbar-start .navbar-link:focus,.navbar.is-black .navbar-start .navbar-link:hover,.navbar.is-black .navbar-start>a.navbar-item.is-active,.navbar.is-black .navbar-start>a.navbar-item:focus,.navbar.is-black .navbar-start>a.navbar-item:hover{background-color:#000;color:#fff}.navbar.is-black .navbar-end .navbar-link::after,.navbar.is-black .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-black .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-black .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-black .navbar-item.has-dropdown:hover .navbar-link{background-color:#000;color:#fff}.navbar.is-black .navbar-dropdown a.navbar-item.is-active{background-color:#0a0a0a;color:#fff}}.navbar.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link,.navbar.is-light .navbar-brand>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link.is-active,.navbar.is-light .navbar-brand .navbar-link:focus,.navbar.is-light .navbar-brand .navbar-link:hover,.navbar.is-light .navbar-brand>a.navbar-item.is-active,.navbar.is-light .navbar-brand>a.navbar-item:focus,.navbar.is-light .navbar-brand>a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-light .navbar-burger{color:rgba(0,0,0,.7)}@media screen and (min-width:1024px){.navbar.is-light .navbar-end .navbar-link,.navbar.is-light .navbar-end>.navbar-item,.navbar.is-light .navbar-start .navbar-link,.navbar.is-light .navbar-start>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-light .navbar-end .navbar-link.is-active,.navbar.is-light .navbar-end .navbar-link:focus,.navbar.is-light .navbar-end .navbar-link:hover,.navbar.is-light .navbar-end>a.navbar-item.is-active,.navbar.is-light .navbar-end>a.navbar-item:focus,.navbar.is-light .navbar-end>a.navbar-item:hover,.navbar.is-light .navbar-start .navbar-link.is-active,.navbar.is-light .navbar-start .navbar-link:focus,.navbar.is-light .navbar-start .navbar-link:hover,.navbar.is-light .navbar-start>a.navbar-item.is-active,.navbar.is-light .navbar-start>a.navbar-item:focus,.navbar.is-light .navbar-start>a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-end .navbar-link::after,.navbar.is-light .navbar-start .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-light .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-light .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-light .navbar-item.has-dropdown:hover .navbar-link{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:rgba(0,0,0,.7)}}.navbar.is-dark{background-color:#363636;color:#fff}.navbar.is-dark .navbar-brand .navbar-link,.navbar.is-dark .navbar-brand>.navbar-item{color:#fff}.navbar.is-dark .navbar-brand .navbar-link.is-active,.navbar.is-dark .navbar-brand .navbar-link:focus,.navbar.is-dark .navbar-brand .navbar-link:hover,.navbar.is-dark .navbar-brand>a.navbar-item.is-active,.navbar.is-dark .navbar-brand>a.navbar-item:focus,.navbar.is-dark .navbar-brand>a.navbar-item:hover{background-color:#292929;color:#fff}.navbar.is-dark .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-dark .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-dark .navbar-end .navbar-link,.navbar.is-dark .navbar-end>.navbar-item,.navbar.is-dark .navbar-start .navbar-link,.navbar.is-dark .navbar-start>.navbar-item{color:#fff}.navbar.is-dark .navbar-end .navbar-link.is-active,.navbar.is-dark .navbar-end .navbar-link:focus,.navbar.is-dark .navbar-end .navbar-link:hover,.navbar.is-dark .navbar-end>a.navbar-item.is-active,.navbar.is-dark .navbar-end>a.navbar-item:focus,.navbar.is-dark .navbar-end>a.navbar-item:hover,.navbar.is-dark .navbar-start .navbar-link.is-active,.navbar.is-dark .navbar-start .navbar-link:focus,.navbar.is-dark .navbar-start .navbar-link:hover,.navbar.is-dark .navbar-start>a.navbar-item.is-active,.navbar.is-dark .navbar-start>a.navbar-item:focus,.navbar.is-dark .navbar-start>a.navbar-item:hover{background-color:#292929;color:#fff}.navbar.is-dark .navbar-end .navbar-link::after,.navbar.is-dark .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-dark .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-dark .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-dark .navbar-item.has-dropdown:hover .navbar-link{background-color:#292929;color:#fff}.navbar.is-dark .navbar-dropdown a.navbar-item.is-active{background-color:#363636;color:#fff}}.navbar.is-primary{background-color:#00d1b2;color:#fff}.navbar.is-primary .navbar-brand .navbar-link,.navbar.is-primary .navbar-brand>.navbar-item{color:#fff}.navbar.is-primary .navbar-brand .navbar-link.is-active,.navbar.is-primary .navbar-brand .navbar-link:focus,.navbar.is-primary .navbar-brand .navbar-link:hover,.navbar.is-primary .navbar-brand>a.navbar-item.is-active,.navbar.is-primary .navbar-brand>a.navbar-item:focus,.navbar.is-primary .navbar-brand>a.navbar-item:hover{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-primary .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-primary .navbar-end .navbar-link,.navbar.is-primary .navbar-end>.navbar-item,.navbar.is-primary .navbar-start .navbar-link,.navbar.is-primary .navbar-start>.navbar-item{color:#fff}.navbar.is-primary .navbar-end .navbar-link.is-active,.navbar.is-primary .navbar-end .navbar-link:focus,.navbar.is-primary .navbar-end .navbar-link:hover,.navbar.is-primary .navbar-end>a.navbar-item.is-active,.navbar.is-primary .navbar-end>a.navbar-item:focus,.navbar.is-primary .navbar-end>a.navbar-item:hover,.navbar.is-primary .navbar-start .navbar-link.is-active,.navbar.is-primary .navbar-start .navbar-link:focus,.navbar.is-primary .navbar-start .navbar-link:hover,.navbar.is-primary .navbar-start>a.navbar-item.is-active,.navbar.is-primary .navbar-start>a.navbar-item:focus,.navbar.is-primary .navbar-start>a.navbar-item:hover{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-end .navbar-link::after,.navbar.is-primary .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-primary .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-primary .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-primary .navbar-item.has-dropdown:hover .navbar-link{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-dropdown a.navbar-item.is-active{background-color:#00d1b2;color:#fff}}.navbar.is-link{background-color:#3273dc;color:#fff}.navbar.is-link .navbar-brand .navbar-link,.navbar.is-link .navbar-brand>.navbar-item{color:#fff}.navbar.is-link .navbar-brand .navbar-link.is-active,.navbar.is-link .navbar-brand .navbar-link:focus,.navbar.is-link .navbar-brand .navbar-link:hover,.navbar.is-link .navbar-brand>a.navbar-item.is-active,.navbar.is-link .navbar-brand>a.navbar-item:focus,.navbar.is-link .navbar-brand>a.navbar-item:hover{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-link .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-link .navbar-end .navbar-link,.navbar.is-link .navbar-end>.navbar-item,.navbar.is-link .navbar-start .navbar-link,.navbar.is-link .navbar-start>.navbar-item{color:#fff}.navbar.is-link .navbar-end .navbar-link.is-active,.navbar.is-link .navbar-end .navbar-link:focus,.navbar.is-link .navbar-end .navbar-link:hover,.navbar.is-link .navbar-end>a.navbar-item.is-active,.navbar.is-link .navbar-end>a.navbar-item:focus,.navbar.is-link .navbar-end>a.navbar-item:hover,.navbar.is-link .navbar-start .navbar-link.is-active,.navbar.is-link .navbar-start .navbar-link:focus,.navbar.is-link .navbar-start .navbar-link:hover,.navbar.is-link .navbar-start>a.navbar-item.is-active,.navbar.is-link .navbar-start>a.navbar-item:focus,.navbar.is-link .navbar-start>a.navbar-item:hover{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-end .navbar-link::after,.navbar.is-link .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-link .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-link .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-link .navbar-item.has-dropdown:hover .navbar-link{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-dropdown a.navbar-item.is-active{background-color:#3273dc;color:#fff}}.navbar.is-info{background-color:#3298dc;color:#fff}.navbar.is-info .navbar-brand .navbar-link,.navbar.is-info .navbar-brand>.navbar-item{color:#fff}.navbar.is-info .navbar-brand .navbar-link.is-active,.navbar.is-info .navbar-brand .navbar-link:focus,.navbar.is-info .navbar-brand .navbar-link:hover,.navbar.is-info .navbar-brand>a.navbar-item.is-active,.navbar.is-info .navbar-brand>a.navbar-item:focus,.navbar.is-info .navbar-brand>a.navbar-item:hover{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-info .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-info .navbar-end .navbar-link,.navbar.is-info .navbar-end>.navbar-item,.navbar.is-info .navbar-start .navbar-link,.navbar.is-info .navbar-start>.navbar-item{color:#fff}.navbar.is-info .navbar-end .navbar-link.is-active,.navbar.is-info .navbar-end .navbar-link:focus,.navbar.is-info .navbar-end .navbar-link:hover,.navbar.is-info .navbar-end>a.navbar-item.is-active,.navbar.is-info .navbar-end>a.navbar-item:focus,.navbar.is-info .navbar-end>a.navbar-item:hover,.navbar.is-info .navbar-start .navbar-link.is-active,.navbar.is-info .navbar-start .navbar-link:focus,.navbar.is-info .navbar-start .navbar-link:hover,.navbar.is-info .navbar-start>a.navbar-item.is-active,.navbar.is-info .navbar-start>a.navbar-item:focus,.navbar.is-info .navbar-start>a.navbar-item:hover{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-end .navbar-link::after,.navbar.is-info .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-info .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-info .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-info .navbar-item.has-dropdown:hover .navbar-link{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-dropdown a.navbar-item.is-active{background-color:#3298dc;color:#fff}}.navbar.is-success{background-color:#48c774;color:#fff}.navbar.is-success .navbar-brand .navbar-link,.navbar.is-success .navbar-brand>.navbar-item{color:#fff}.navbar.is-success .navbar-brand .navbar-link.is-active,.navbar.is-success .navbar-brand .navbar-link:focus,.navbar.is-success .navbar-brand .navbar-link:hover,.navbar.is-success .navbar-brand>a.navbar-item.is-active,.navbar.is-success .navbar-brand>a.navbar-item:focus,.navbar.is-success .navbar-brand>a.navbar-item:hover{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-success .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-success .navbar-end .navbar-link,.navbar.is-success .navbar-end>.navbar-item,.navbar.is-success .navbar-start .navbar-link,.navbar.is-success .navbar-start>.navbar-item{color:#fff}.navbar.is-success .navbar-end .navbar-link.is-active,.navbar.is-success .navbar-end .navbar-link:focus,.navbar.is-success .navbar-end .navbar-link:hover,.navbar.is-success .navbar-end>a.navbar-item.is-active,.navbar.is-success .navbar-end>a.navbar-item:focus,.navbar.is-success .navbar-end>a.navbar-item:hover,.navbar.is-success .navbar-start .navbar-link.is-active,.navbar.is-success .navbar-start .navbar-link:focus,.navbar.is-success .navbar-start .navbar-link:hover,.navbar.is-success .navbar-start>a.navbar-item.is-active,.navbar.is-success .navbar-start>a.navbar-item:focus,.navbar.is-success .navbar-start>a.navbar-item:hover{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-end .navbar-link::after,.navbar.is-success .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-success .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-success .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-success .navbar-item.has-dropdown:hover .navbar-link{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-dropdown a.navbar-item.is-active{background-color:#48c774;color:#fff}}.navbar.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link,.navbar.is-warning .navbar-brand>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link.is-active,.navbar.is-warning .navbar-brand .navbar-link:focus,.navbar.is-warning .navbar-brand .navbar-link:hover,.navbar.is-warning .navbar-brand>a.navbar-item.is-active,.navbar.is-warning .navbar-brand>a.navbar-item:focus,.navbar.is-warning .navbar-brand>a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-burger{color:rgba(0,0,0,.7)}@media screen and (min-width:1024px){.navbar.is-warning .navbar-end .navbar-link,.navbar.is-warning .navbar-end>.navbar-item,.navbar.is-warning .navbar-start .navbar-link,.navbar.is-warning .navbar-start>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-end .navbar-link.is-active,.navbar.is-warning .navbar-end .navbar-link:focus,.navbar.is-warning .navbar-end .navbar-link:hover,.navbar.is-warning .navbar-end>a.navbar-item.is-active,.navbar.is-warning .navbar-end>a.navbar-item:focus,.navbar.is-warning .navbar-end>a.navbar-item:hover,.navbar.is-warning .navbar-start .navbar-link.is-active,.navbar.is-warning .navbar-start .navbar-link:focus,.navbar.is-warning .navbar-start .navbar-link:hover,.navbar.is-warning .navbar-start>a.navbar-item.is-active,.navbar.is-warning .navbar-start>a.navbar-item:focus,.navbar.is-warning .navbar-start>a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-end .navbar-link::after,.navbar.is-warning .navbar-start .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-warning .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-warning .navbar-item.has-dropdown:hover .navbar-link{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-dropdown a.navbar-item.is-active{background-color:#ffdd57;color:rgba(0,0,0,.7)}}.navbar.is-danger{background-color:#f14668;color:#fff}.navbar.is-danger .navbar-brand .navbar-link,.navbar.is-danger .navbar-brand>.navbar-item{color:#fff}.navbar.is-danger .navbar-brand .navbar-link.is-active,.navbar.is-danger .navbar-brand .navbar-link:focus,.navbar.is-danger .navbar-brand .navbar-link:hover,.navbar.is-danger .navbar-brand>a.navbar-item.is-active,.navbar.is-danger .navbar-brand>a.navbar-item:focus,.navbar.is-danger .navbar-brand>a.navbar-item:hover{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-danger .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-danger .navbar-end .navbar-link,.navbar.is-danger .navbar-end>.navbar-item,.navbar.is-danger .navbar-start .navbar-link,.navbar.is-danger .navbar-start>.navbar-item{color:#fff}.navbar.is-danger .navbar-end .navbar-link.is-active,.navbar.is-danger .navbar-end .navbar-link:focus,.navbar.is-danger .navbar-end .navbar-link:hover,.navbar.is-danger .navbar-end>a.navbar-item.is-active,.navbar.is-danger .navbar-end>a.navbar-item:focus,.navbar.is-danger .navbar-end>a.navbar-item:hover,.navbar.is-danger .navbar-start .navbar-link.is-active,.navbar.is-danger .navbar-start .navbar-link:focus,.navbar.is-danger .navbar-start .navbar-link:hover,.navbar.is-danger .navbar-start>a.navbar-item.is-active,.navbar.is-danger .navbar-start>a.navbar-item:focus,.navbar.is-danger .navbar-start>a.navbar-item:hover{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-end .navbar-link::after,.navbar.is-danger .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-danger .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-danger .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-danger .navbar-item.has-dropdown:hover .navbar-link{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-dropdown a.navbar-item.is-active{background-color:#f14668;color:#fff}}.navbar>.container{align-items:stretch;display:flex;min-height:3.25rem;width:100%}.navbar.has-shadow{box-shadow:0 2px 0 0 #f5f5f5}.navbar.is-fixed-bottom,.navbar.is-fixed-top{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom{bottom:0}.navbar.is-fixed-bottom.has-shadow{box-shadow:0 -2px 0 0 #f5f5f5}.navbar.is-fixed-top{top:0}body.has-navbar-fixed-top,html.has-navbar-fixed-top{padding-top:3.25rem}body.has-navbar-fixed-bottom,html.has-navbar-fixed-bottom{padding-bottom:3.25rem}.navbar-brand,.navbar-tabs{align-items:stretch;display:flex;flex-shrink:0;min-height:3.25rem}.navbar-brand a.navbar-item:focus,.navbar-brand a.navbar-item:hover{background-color:transparent}.navbar-tabs{-webkit-overflow-scrolling:touch;max-width:100vw;overflow-x:auto;overflow-y:hidden}.navbar-burger{color:#4a4a4a;cursor:pointer;display:block;height:3.25rem;position:relative;width:3.25rem;margin-left:auto}.navbar-burger span{background-color:currentColor;display:block;height:1px;left:calc(50% - 8px);position:absolute;transform-origin:center;transition-duration:86ms;transition-property:background-color,opacity,transform;transition-timing-function:ease-out;width:16px}.navbar-burger span:nth-child(1){top:calc(50% - 6px)}.navbar-burger span:nth-child(2){top:calc(50% - 1px)}.navbar-burger span:nth-child(3){top:calc(50% + 4px)}.navbar-burger:hover{background-color:rgba(0,0,0,.05)}.navbar-burger.is-active span:nth-child(1){transform:translateY(5px) rotate(45deg)}.navbar-burger.is-active span:nth-child(2){opacity:0}.navbar-burger.is-active span:nth-child(3){transform:translateY(-5px) rotate(-45deg)}.navbar-menu{display:none}.navbar-item,.navbar-link{color:#4a4a4a;display:block;line-height:1.5;padding:.5rem .75rem;position:relative}.navbar-item .icon:only-child,.navbar-link .icon:only-child{margin-left:-.25rem;margin-right:-.25rem}.navbar-link,a.navbar-item{cursor:pointer}.navbar-link.is-active,.navbar-link:focus,.navbar-link:focus-within,.navbar-link:hover,a.navbar-item.is-active,a.navbar-item:focus,a.navbar-item:focus-within,a.navbar-item:hover{background-color:#fafafa;color:#3273dc}.navbar-item{display:block;flex-grow:0;flex-shrink:0}.navbar-item img{max-height:1.75rem}.navbar-item.has-dropdown{padding:0}.navbar-item.is-expanded{flex-grow:1;flex-shrink:1}.navbar-item.is-tab{border-bottom:1px solid transparent;min-height:3.25rem;padding-bottom:calc(.5rem - 1px)}.navbar-item.is-tab:focus,.navbar-item.is-tab:hover{background-color:transparent;border-bottom-color:#3273dc}.navbar-item.is-tab.is-active{background-color:transparent;border-bottom-color:#3273dc;border-bottom-style:solid;border-bottom-width:3px;color:#3273dc;padding-bottom:calc(.5rem - 3px)}.navbar-content{flex-grow:1;flex-shrink:1}.navbar-link:not(.is-arrowless){padding-right:2.5em}.navbar-link:not(.is-arrowless)::after{border-color:#3273dc;margin-top:-.375em;right:1.125em}.navbar-dropdown{font-size:.875rem;padding-bottom:.5rem;padding-top:.5rem}.navbar-dropdown .navbar-item{padding-left:1.5rem;padding-right:1.5rem}.navbar-divider{background-color:#f5f5f5;border:none;display:none;height:2px;margin:.5rem 0}@media screen and (max-width:1023px){.navbar>.container{display:block}.navbar-brand .navbar-item,.navbar-tabs .navbar-item{align-items:center;display:flex}.navbar-link::after{display:none}.navbar-menu{background-color:#fff;box-shadow:0 8px 16px rgba(10,10,10,.1);padding:.5rem 0}.navbar-menu.is-active{display:block}.navbar.is-fixed-bottom-touch,.navbar.is-fixed-top-touch{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom-touch{bottom:0}.navbar.is-fixed-bottom-touch.has-shadow{box-shadow:0 -2px 3px rgba(10,10,10,.1)}.navbar.is-fixed-top-touch{top:0}.navbar.is-fixed-top .navbar-menu,.navbar.is-fixed-top-touch .navbar-menu{-webkit-overflow-scrolling:touch;max-height:calc(100vh - 3.25rem);overflow:auto}body.has-navbar-fixed-top-touch,html.has-navbar-fixed-top-touch{padding-top:3.25rem}body.has-navbar-fixed-bottom-touch,html.has-navbar-fixed-bottom-touch{padding-bottom:3.25rem}}@media screen and (min-width:1024px){.navbar,.navbar-end,.navbar-menu,.navbar-start{align-items:stretch;display:flex}.navbar{min-height:3.25rem}.navbar.is-spaced{padding:1rem 2rem}.navbar.is-spaced .navbar-end,.navbar.is-spaced .navbar-start{align-items:center}.navbar.is-spaced .navbar-link,.navbar.is-spaced a.navbar-item{border-radius:4px}.navbar.is-transparent .navbar-link.is-active,.navbar.is-transparent .navbar-link:focus,.navbar.is-transparent .navbar-link:hover,.navbar.is-transparent a.navbar-item.is-active,.navbar.is-transparent a.navbar-item:focus,.navbar.is-transparent a.navbar-item:hover{background-color:transparent!important}.navbar.is-transparent .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:focus .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:focus-within .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:hover .navbar-link{background-color:transparent!important}.navbar.is-transparent .navbar-dropdown a.navbar-item:focus,.navbar.is-transparent .navbar-dropdown a.navbar-item:hover{background-color:#f5f5f5;color:#0a0a0a}.navbar.is-transparent .navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:#3273dc}.navbar-burger{display:none}.navbar-item,.navbar-link{align-items:center;display:flex}.navbar-item{display:flex}.navbar-item.has-dropdown{align-items:stretch}.navbar-item.has-dropdown-up .navbar-link::after{transform:rotate(135deg) translate(.25em,-.25em)}.navbar-item.has-dropdown-up .navbar-dropdown{border-bottom:2px solid #dbdbdb;border-radius:6px 6px 0 0;border-top:none;bottom:100%;box-shadow:0 -8px 8px rgba(10,10,10,.1);top:auto}.navbar-item.is-active .navbar-dropdown,.navbar-item.is-hoverable:focus .navbar-dropdown,.navbar-item.is-hoverable:focus-within .navbar-dropdown,.navbar-item.is-hoverable:hover .navbar-dropdown{display:block}.navbar-item.is-active .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:focus .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:focus-within .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:hover .navbar-dropdown.is-boxed,.navbar.is-spaced .navbar-item.is-active .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:focus .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:focus-within .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:hover .navbar-dropdown{opacity:1;pointer-events:auto;transform:translateY(0)}.navbar-menu{flex-grow:1;flex-shrink:0}.navbar-start{justify-content:flex-start;margin-right:auto}.navbar-end{justify-content:flex-end;margin-left:auto}.navbar-dropdown{background-color:#fff;border-bottom-left-radius:6px;border-bottom-right-radius:6px;border-top:2px solid #dbdbdb;box-shadow:0 8px 8px rgba(10,10,10,.1);display:none;font-size:.875rem;left:0;min-width:100%;position:absolute;top:100%;z-index:20}.navbar-dropdown .navbar-item{padding:.375rem 1rem;white-space:nowrap}.navbar-dropdown a.navbar-item{padding-right:3rem}.navbar-dropdown a.navbar-item:focus,.navbar-dropdown a.navbar-item:hover{background-color:#f5f5f5;color:#0a0a0a}.navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:#3273dc}.navbar-dropdown.is-boxed,.navbar.is-spaced .navbar-dropdown{border-radius:6px;border-top:none;box-shadow:0 8px 8px rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.1);display:block;opacity:0;pointer-events:none;top:calc(100% + (-4px));transform:translateY(-5px);transition-duration:86ms;transition-property:opacity,transform}.navbar-dropdown.is-right{left:auto;right:0}.navbar-divider{display:block}.container>.navbar .navbar-brand,.navbar>.container .navbar-brand{margin-left:-.75rem}.container>.navbar .navbar-menu,.navbar>.container .navbar-menu{margin-right:-.75rem}.navbar.is-fixed-bottom-desktop,.navbar.is-fixed-top-desktop{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom-desktop{bottom:0}.navbar.is-fixed-bottom-desktop.has-shadow{box-shadow:0 -2px 3px rgba(10,10,10,.1)}.navbar.is-fixed-top-desktop{top:0}body.has-navbar-fixed-top-desktop,html.has-navbar-fixed-top-desktop{padding-top:3.25rem}body.has-navbar-fixed-bottom-desktop,html.has-navbar-fixed-bottom-desktop{padding-bottom:3.25rem}body.has-spaced-navbar-fixed-top,html.has-spaced-navbar-fixed-top{padding-top:5.25rem}body.has-spaced-navbar-fixed-bottom,html.has-spaced-navbar-fixed-bottom{padding-bottom:5.25rem}.navbar-link.is-active,a.navbar-item.is-active{color:#0a0a0a}.navbar-link.is-active:not(:focus):not(:hover),a.navbar-item.is-active:not(:focus):not(:hover){background-color:transparent}.navbar-item.has-dropdown.is-active .navbar-link,.navbar-item.has-dropdown:focus .navbar-link,.navbar-item.has-dropdown:hover .navbar-link{background-color:#fafafa}}.hero.is-fullheight-with-navbar{min-height:calc(100vh - 3.25rem)}.pagination{font-size:1rem;margin:-.25rem}.pagination.is-small{font-size:.75rem}.pagination.is-medium{font-size:1.25rem}.pagination.is-large{font-size:1.5rem}.pagination.is-rounded .pagination-next,.pagination.is-rounded .pagination-previous{padding-left:1em;padding-right:1em;border-radius:290486px}.pagination.is-rounded .pagination-link{border-radius:290486px}.pagination,.pagination-list{align-items:center;display:flex;justify-content:center;text-align:center}.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous{font-size:1em;justify-content:center;margin:.25rem;padding-left:.5em;padding-right:.5em;text-align:center}.pagination-link,.pagination-next,.pagination-previous{border-color:#dbdbdb;color:#363636;min-width:2.5em}.pagination-link:hover,.pagination-next:hover,.pagination-previous:hover{border-color:#b5b5b5;color:#363636}.pagination-link:focus,.pagination-next:focus,.pagination-previous:focus{border-color:#3273dc}.pagination-link:active,.pagination-next:active,.pagination-previous:active{box-shadow:inset 0 1px 2px rgba(10,10,10,.2)}.pagination-link[disabled],.pagination-next[disabled],.pagination-previous[disabled]{background-color:#dbdbdb;border-color:#dbdbdb;box-shadow:none;color:#7a7a7a;opacity:.5}.pagination-next,.pagination-previous{padding-left:.75em;padding-right:.75em;white-space:nowrap}.pagination-link.is-current{background-color:#3273dc;border-color:#3273dc;color:#fff}.pagination-ellipsis{color:#b5b5b5;pointer-events:none}.pagination-list{flex-wrap:wrap}@media screen and (max-width:768px){.pagination{flex-wrap:wrap}.pagination-next,.pagination-previous{flex-grow:1;flex-shrink:1}.pagination-list li{flex-grow:1;flex-shrink:1}}@media screen and (min-width:769px),print{.pagination-list{flex-grow:1;flex-shrink:1;justify-content:flex-start;order:1}.pagination-previous{order:2}.pagination-next{order:3}.pagination{justify-content:space-between}.pagination.is-centered .pagination-previous{order:1}.pagination.is-centered .pagination-list{justify-content:center;order:2}.pagination.is-centered .pagination-next{order:3}.pagination.is-right .pagination-previous{order:1}.pagination.is-right .pagination-next{order:2}.pagination.is-right .pagination-list{justify-content:flex-end;order:3}}.panel{border-radius:6px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);font-size:1rem}.panel:not(:last-child){margin-bottom:1.5rem}.panel.is-white .panel-heading{background-color:#fff;color:#0a0a0a}.panel.is-white .panel-tabs a.is-active{border-bottom-color:#fff}.panel.is-white .panel-block.is-active .panel-icon{color:#fff}.panel.is-black .panel-heading{background-color:#0a0a0a;color:#fff}.panel.is-black .panel-tabs a.is-active{border-bottom-color:#0a0a0a}.panel.is-black .panel-block.is-active .panel-icon{color:#0a0a0a}.panel.is-light .panel-heading{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.panel.is-light .panel-tabs a.is-active{border-bottom-color:#f5f5f5}.panel.is-light .panel-block.is-active .panel-icon{color:#f5f5f5}.panel.is-dark .panel-heading{background-color:#363636;color:#fff}.panel.is-dark .panel-tabs a.is-active{border-bottom-color:#363636}.panel.is-dark .panel-block.is-active .panel-icon{color:#363636}.panel.is-primary .panel-heading{background-color:#00d1b2;color:#fff}.panel.is-primary .panel-tabs a.is-active{border-bottom-color:#00d1b2}.panel.is-primary .panel-block.is-active .panel-icon{color:#00d1b2}.panel.is-link .panel-heading{background-color:#3273dc;color:#fff}.panel.is-link .panel-tabs a.is-active{border-bottom-color:#3273dc}.panel.is-link .panel-block.is-active .panel-icon{color:#3273dc}.panel.is-info .panel-heading{background-color:#3298dc;color:#fff}.panel.is-info .panel-tabs a.is-active{border-bottom-color:#3298dc}.panel.is-info .panel-block.is-active .panel-icon{color:#3298dc}.panel.is-success .panel-heading{background-color:#48c774;color:#fff}.panel.is-success .panel-tabs a.is-active{border-bottom-color:#48c774}.panel.is-success .panel-block.is-active .panel-icon{color:#48c774}.panel.is-warning .panel-heading{background-color:#ffdd57;color:rgba(0,0,0,.7)}.panel.is-warning .panel-tabs a.is-active{border-bottom-color:#ffdd57}.panel.is-warning .panel-block.is-active .panel-icon{color:#ffdd57}.panel.is-danger .panel-heading{background-color:#f14668;color:#fff}.panel.is-danger .panel-tabs a.is-active{border-bottom-color:#f14668}.panel.is-danger .panel-block.is-active .panel-icon{color:#f14668}.panel-block:not(:last-child),.panel-tabs:not(:last-child){border-bottom:1px solid #ededed}.panel-heading{background-color:#ededed;border-radius:6px 6px 0 0;color:#363636;font-size:1.25em;font-weight:700;line-height:1.25;padding:.75em 1em}.panel-tabs{align-items:flex-end;display:flex;font-size:.875em;justify-content:center}.panel-tabs a{border-bottom:1px solid #dbdbdb;margin-bottom:-1px;padding:.5em}.panel-tabs a.is-active{border-bottom-color:#4a4a4a;color:#363636}.panel-list a{color:#4a4a4a}.panel-list a:hover{color:#3273dc}.panel-block{align-items:center;color:#363636;display:flex;justify-content:flex-start;padding:.5em .75em}.panel-block input[type=checkbox]{margin-right:.75em}.panel-block>.control{flex-grow:1;flex-shrink:1;width:100%}.panel-block.is-wrapped{flex-wrap:wrap}.panel-block.is-active{border-left-color:#3273dc;color:#363636}.panel-block.is-active .panel-icon{color:#3273dc}.panel-block:last-child{border-bottom-left-radius:6px;border-bottom-right-radius:6px}a.panel-block,label.panel-block{cursor:pointer}a.panel-block:hover,label.panel-block:hover{background-color:#f5f5f5}.panel-icon{display:inline-block;font-size:14px;height:1em;line-height:1em;text-align:center;vertical-align:top;width:1em;color:#7a7a7a;margin-right:.75em}.panel-icon .fa{font-size:inherit;line-height:inherit}.tabs{-webkit-overflow-scrolling:touch;align-items:stretch;display:flex;font-size:1rem;justify-content:space-between;overflow:hidden;overflow-x:auto;white-space:nowrap}.tabs a{align-items:center;border-bottom-color:#dbdbdb;border-bottom-style:solid;border-bottom-width:1px;color:#4a4a4a;display:flex;justify-content:center;margin-bottom:-1px;padding:.5em 1em;vertical-align:top}.tabs a:hover{border-bottom-color:#363636;color:#363636}.tabs li{display:block}.tabs li.is-active a{border-bottom-color:#3273dc;color:#3273dc}.tabs ul{align-items:center;border-bottom-color:#dbdbdb;border-bottom-style:solid;border-bottom-width:1px;display:flex;flex-grow:1;flex-shrink:0;justify-content:flex-start}.tabs ul.is-left{padding-right:.75em}.tabs ul.is-center{flex:none;justify-content:center;padding-left:.75em;padding-right:.75em}.tabs ul.is-right{justify-content:flex-end;padding-left:.75em}.tabs .icon:first-child{margin-right:.5em}.tabs .icon:last-child{margin-left:.5em}.tabs.is-centered ul{justify-content:center}.tabs.is-right ul{justify-content:flex-end}.tabs.is-boxed a{border:1px solid transparent;border-radius:4px 4px 0 0}.tabs.is-boxed a:hover{background-color:#f5f5f5;border-bottom-color:#dbdbdb}.tabs.is-boxed li.is-active a{background-color:#fff;border-color:#dbdbdb;border-bottom-color:transparent!important}.tabs.is-fullwidth li{flex-grow:1;flex-shrink:0}.tabs.is-toggle a{border-color:#dbdbdb;border-style:solid;border-width:1px;margin-bottom:0;position:relative}.tabs.is-toggle a:hover{background-color:#f5f5f5;border-color:#b5b5b5;z-index:2}.tabs.is-toggle li+li{margin-left:-1px}.tabs.is-toggle li:first-child a{border-radius:4px 0 0 4px}.tabs.is-toggle li:last-child a{border-radius:0 4px 4px 0}.tabs.is-toggle li.is-active a{background-color:#3273dc;border-color:#3273dc;color:#fff;z-index:1}.tabs.is-toggle ul{border-bottom:none}.tabs.is-toggle.is-toggle-rounded li:first-child a{border-bottom-left-radius:290486px;border-top-left-radius:290486px;padding-left:1.25em}.tabs.is-toggle.is-toggle-rounded li:last-child a{border-bottom-right-radius:290486px;border-top-right-radius:290486px;padding-right:1.25em}.tabs.is-small{font-size:.75rem}.tabs.is-medium{font-size:1.25rem}.tabs.is-large{font-size:1.5rem}.column{display:block;flex-basis:0;flex-grow:1;flex-shrink:1;padding:.75rem}.columns.is-mobile>.column.is-narrow{flex:none}.columns.is-mobile>.column.is-full{flex:none;width:100%}.columns.is-mobile>.column.is-three-quarters{flex:none;width:75%}.columns.is-mobile>.column.is-two-thirds{flex:none;width:66.6666%}.columns.is-mobile>.column.is-half{flex:none;width:50%}.columns.is-mobile>.column.is-one-third{flex:none;width:33.3333%}.columns.is-mobile>.column.is-one-quarter{flex:none;width:25%}.columns.is-mobile>.column.is-one-fifth{flex:none;width:20%}.columns.is-mobile>.column.is-two-fifths{flex:none;width:40%}.columns.is-mobile>.column.is-three-fifths{flex:none;width:60%}.columns.is-mobile>.column.is-four-fifths{flex:none;width:80%}.columns.is-mobile>.column.is-offset-three-quarters{margin-left:75%}.columns.is-mobile>.column.is-offset-two-thirds{margin-left:66.6666%}.columns.is-mobile>.column.is-offset-half{margin-left:50%}.columns.is-mobile>.column.is-offset-one-third{margin-left:33.3333%}.columns.is-mobile>.column.is-offset-one-quarter{margin-left:25%}.columns.is-mobile>.column.is-offset-one-fifth{margin-left:20%}.columns.is-mobile>.column.is-offset-two-fifths{margin-left:40%}.columns.is-mobile>.column.is-offset-three-fifths{margin-left:60%}.columns.is-mobile>.column.is-offset-four-fifths{margin-left:80%}.columns.is-mobile>.column.is-0{flex:none;width:0%}.columns.is-mobile>.column.is-offset-0{margin-left:0}.columns.is-mobile>.column.is-1{flex:none;width:8.33333%}.columns.is-mobile>.column.is-offset-1{margin-left:8.33333%}.columns.is-mobile>.column.is-2{flex:none;width:16.66667%}.columns.is-mobile>.column.is-offset-2{margin-left:16.66667%}.columns.is-mobile>.column.is-3{flex:none;width:25%}.columns.is-mobile>.column.is-offset-3{margin-left:25%}.columns.is-mobile>.column.is-4{flex:none;width:33.33333%}.columns.is-mobile>.column.is-offset-4{margin-left:33.33333%}.columns.is-mobile>.column.is-5{flex:none;width:41.66667%}.columns.is-mobile>.column.is-offset-5{margin-left:41.66667%}.columns.is-mobile>.column.is-6{flex:none;width:50%}.columns.is-mobile>.column.is-offset-6{margin-left:50%}.columns.is-mobile>.column.is-7{flex:none;width:58.33333%}.columns.is-mobile>.column.is-offset-7{margin-left:58.33333%}.columns.is-mobile>.column.is-8{flex:none;width:66.66667%}.columns.is-mobile>.column.is-offset-8{margin-left:66.66667%}.columns.is-mobile>.column.is-9{flex:none;width:75%}.columns.is-mobile>.column.is-offset-9{margin-left:75%}.columns.is-mobile>.column.is-10{flex:none;width:83.33333%}.columns.is-mobile>.column.is-offset-10{margin-left:83.33333%}.columns.is-mobile>.column.is-11{flex:none;width:91.66667%}.columns.is-mobile>.column.is-offset-11{margin-left:91.66667%}.columns.is-mobile>.column.is-12{flex:none;width:100%}.columns.is-mobile>.column.is-offset-12{margin-left:100%}@media screen and (max-width:768px){.column.is-narrow-mobile{flex:none}.column.is-full-mobile{flex:none;width:100%}.column.is-three-quarters-mobile{flex:none;width:75%}.column.is-two-thirds-mobile{flex:none;width:66.6666%}.column.is-half-mobile{flex:none;width:50%}.column.is-one-third-mobile{flex:none;width:33.3333%}.column.is-one-quarter-mobile{flex:none;width:25%}.column.is-one-fifth-mobile{flex:none;width:20%}.column.is-two-fifths-mobile{flex:none;width:40%}.column.is-three-fifths-mobile{flex:none;width:60%}.column.is-four-fifths-mobile{flex:none;width:80%}.column.is-offset-three-quarters-mobile{margin-left:75%}.column.is-offset-two-thirds-mobile{margin-left:66.6666%}.column.is-offset-half-mobile{margin-left:50%}.column.is-offset-one-third-mobile{margin-left:33.3333%}.column.is-offset-one-quarter-mobile{margin-left:25%}.column.is-offset-one-fifth-mobile{margin-left:20%}.column.is-offset-two-fifths-mobile{margin-left:40%}.column.is-offset-three-fifths-mobile{margin-left:60%}.column.is-offset-four-fifths-mobile{margin-left:80%}.column.is-0-mobile{flex:none;width:0%}.column.is-offset-0-mobile{margin-left:0}.column.is-1-mobile{flex:none;width:8.33333%}.column.is-offset-1-mobile{margin-left:8.33333%}.column.is-2-mobile{flex:none;width:16.66667%}.column.is-offset-2-mobile{margin-left:16.66667%}.column.is-3-mobile{flex:none;width:25%}.column.is-offset-3-mobile{margin-left:25%}.column.is-4-mobile{flex:none;width:33.33333%}.column.is-offset-4-mobile{margin-left:33.33333%}.column.is-5-mobile{flex:none;width:41.66667%}.column.is-offset-5-mobile{margin-left:41.66667%}.column.is-6-mobile{flex:none;width:50%}.column.is-offset-6-mobile{margin-left:50%}.column.is-7-mobile{flex:none;width:58.33333%}.column.is-offset-7-mobile{margin-left:58.33333%}.column.is-8-mobile{flex:none;width:66.66667%}.column.is-offset-8-mobile{margin-left:66.66667%}.column.is-9-mobile{flex:none;width:75%}.column.is-offset-9-mobile{margin-left:75%}.column.is-10-mobile{flex:none;width:83.33333%}.column.is-offset-10-mobile{margin-left:83.33333%}.column.is-11-mobile{flex:none;width:91.66667%}.column.is-offset-11-mobile{margin-left:91.66667%}.column.is-12-mobile{flex:none;width:100%}.column.is-offset-12-mobile{margin-left:100%}}@media screen and (min-width:769px),print{.column.is-narrow,.column.is-narrow-tablet{flex:none}.column.is-full,.column.is-full-tablet{flex:none;width:100%}.column.is-three-quarters,.column.is-three-quarters-tablet{flex:none;width:75%}.column.is-two-thirds,.column.is-two-thirds-tablet{flex:none;width:66.6666%}.column.is-half,.column.is-half-tablet{flex:none;width:50%}.column.is-one-third,.column.is-one-third-tablet{flex:none;width:33.3333%}.column.is-one-quarter,.column.is-one-quarter-tablet{flex:none;width:25%}.column.is-one-fifth,.column.is-one-fifth-tablet{flex:none;width:20%}.column.is-two-fifths,.column.is-two-fifths-tablet{flex:none;width:40%}.column.is-three-fifths,.column.is-three-fifths-tablet{flex:none;width:60%}.column.is-four-fifths,.column.is-four-fifths-tablet{flex:none;width:80%}.column.is-offset-three-quarters,.column.is-offset-three-quarters-tablet{margin-left:75%}.column.is-offset-two-thirds,.column.is-offset-two-thirds-tablet{margin-left:66.6666%}.column.is-offset-half,.column.is-offset-half-tablet{margin-left:50%}.column.is-offset-one-third,.column.is-offset-one-third-tablet{margin-left:33.3333%}.column.is-offset-one-quarter,.column.is-offset-one-quarter-tablet{margin-left:25%}.column.is-offset-one-fifth,.column.is-offset-one-fifth-tablet{margin-left:20%}.column.is-offset-two-fifths,.column.is-offset-two-fifths-tablet{margin-left:40%}.column.is-offset-three-fifths,.column.is-offset-three-fifths-tablet{margin-left:60%}.column.is-offset-four-fifths,.column.is-offset-four-fifths-tablet{margin-left:80%}.column.is-0,.column.is-0-tablet{flex:none;width:0%}.column.is-offset-0,.column.is-offset-0-tablet{margin-left:0}.column.is-1,.column.is-1-tablet{flex:none;width:8.33333%}.column.is-offset-1,.column.is-offset-1-tablet{margin-left:8.33333%}.column.is-2,.column.is-2-tablet{flex:none;width:16.66667%}.column.is-offset-2,.column.is-offset-2-tablet{margin-left:16.66667%}.column.is-3,.column.is-3-tablet{flex:none;width:25%}.column.is-offset-3,.column.is-offset-3-tablet{margin-left:25%}.column.is-4,.column.is-4-tablet{flex:none;width:33.33333%}.column.is-offset-4,.column.is-offset-4-tablet{margin-left:33.33333%}.column.is-5,.column.is-5-tablet{flex:none;width:41.66667%}.column.is-offset-5,.column.is-offset-5-tablet{margin-left:41.66667%}.column.is-6,.column.is-6-tablet{flex:none;width:50%}.column.is-offset-6,.column.is-offset-6-tablet{margin-left:50%}.column.is-7,.column.is-7-tablet{flex:none;width:58.33333%}.column.is-offset-7,.column.is-offset-7-tablet{margin-left:58.33333%}.column.is-8,.column.is-8-tablet{flex:none;width:66.66667%}.column.is-offset-8,.column.is-offset-8-tablet{margin-left:66.66667%}.column.is-9,.column.is-9-tablet{flex:none;width:75%}.column.is-offset-9,.column.is-offset-9-tablet{margin-left:75%}.column.is-10,.column.is-10-tablet{flex:none;width:83.33333%}.column.is-offset-10,.column.is-offset-10-tablet{margin-left:83.33333%}.column.is-11,.column.is-11-tablet{flex:none;width:91.66667%}.column.is-offset-11,.column.is-offset-11-tablet{margin-left:91.66667%}.column.is-12,.column.is-12-tablet{flex:none;width:100%}.column.is-offset-12,.column.is-offset-12-tablet{margin-left:100%}}@media screen and (max-width:1023px){.column.is-narrow-touch{flex:none}.column.is-full-touch{flex:none;width:100%}.column.is-three-quarters-touch{flex:none;width:75%}.column.is-two-thirds-touch{flex:none;width:66.6666%}.column.is-half-touch{flex:none;width:50%}.column.is-one-third-touch{flex:none;width:33.3333%}.column.is-one-quarter-touch{flex:none;width:25%}.column.is-one-fifth-touch{flex:none;width:20%}.column.is-two-fifths-touch{flex:none;width:40%}.column.is-three-fifths-touch{flex:none;width:60%}.column.is-four-fifths-touch{flex:none;width:80%}.column.is-offset-three-quarters-touch{margin-left:75%}.column.is-offset-two-thirds-touch{margin-left:66.6666%}.column.is-offset-half-touch{margin-left:50%}.column.is-offset-one-third-touch{margin-left:33.3333%}.column.is-offset-one-quarter-touch{margin-left:25%}.column.is-offset-one-fifth-touch{margin-left:20%}.column.is-offset-two-fifths-touch{margin-left:40%}.column.is-offset-three-fifths-touch{margin-left:60%}.column.is-offset-four-fifths-touch{margin-left:80%}.column.is-0-touch{flex:none;width:0%}.column.is-offset-0-touch{margin-left:0}.column.is-1-touch{flex:none;width:8.33333%}.column.is-offset-1-touch{margin-left:8.33333%}.column.is-2-touch{flex:none;width:16.66667%}.column.is-offset-2-touch{margin-left:16.66667%}.column.is-3-touch{flex:none;width:25%}.column.is-offset-3-touch{margin-left:25%}.column.is-4-touch{flex:none;width:33.33333%}.column.is-offset-4-touch{margin-left:33.33333%}.column.is-5-touch{flex:none;width:41.66667%}.column.is-offset-5-touch{margin-left:41.66667%}.column.is-6-touch{flex:none;width:50%}.column.is-offset-6-touch{margin-left:50%}.column.is-7-touch{flex:none;width:58.33333%}.column.is-offset-7-touch{margin-left:58.33333%}.column.is-8-touch{flex:none;width:66.66667%}.column.is-offset-8-touch{margin-left:66.66667%}.column.is-9-touch{flex:none;width:75%}.column.is-offset-9-touch{margin-left:75%}.column.is-10-touch{flex:none;width:83.33333%}.column.is-offset-10-touch{margin-left:83.33333%}.column.is-11-touch{flex:none;width:91.66667%}.column.is-offset-11-touch{margin-left:91.66667%}.column.is-12-touch{flex:none;width:100%}.column.is-offset-12-touch{margin-left:100%}}@media screen and (min-width:1024px){.column.is-narrow-desktop{flex:none}.column.is-full-desktop{flex:none;width:100%}.column.is-three-quarters-desktop{flex:none;width:75%}.column.is-two-thirds-desktop{flex:none;width:66.6666%}.column.is-half-desktop{flex:none;width:50%}.column.is-one-third-desktop{flex:none;width:33.3333%}.column.is-one-quarter-desktop{flex:none;width:25%}.column.is-one-fifth-desktop{flex:none;width:20%}.column.is-two-fifths-desktop{flex:none;width:40%}.column.is-three-fifths-desktop{flex:none;width:60%}.column.is-four-fifths-desktop{flex:none;width:80%}.column.is-offset-three-quarters-desktop{margin-left:75%}.column.is-offset-two-thirds-desktop{margin-left:66.6666%}.column.is-offset-half-desktop{margin-left:50%}.column.is-offset-one-third-desktop{margin-left:33.3333%}.column.is-offset-one-quarter-desktop{margin-left:25%}.column.is-offset-one-fifth-desktop{margin-left:20%}.column.is-offset-two-fifths-desktop{margin-left:40%}.column.is-offset-three-fifths-desktop{margin-left:60%}.column.is-offset-four-fifths-desktop{margin-left:80%}.column.is-0-desktop{flex:none;width:0%}.column.is-offset-0-desktop{margin-left:0}.column.is-1-desktop{flex:none;width:8.33333%}.column.is-offset-1-desktop{margin-left:8.33333%}.column.is-2-desktop{flex:none;width:16.66667%}.column.is-offset-2-desktop{margin-left:16.66667%}.column.is-3-desktop{flex:none;width:25%}.column.is-offset-3-desktop{margin-left:25%}.column.is-4-desktop{flex:none;width:33.33333%}.column.is-offset-4-desktop{margin-left:33.33333%}.column.is-5-desktop{flex:none;width:41.66667%}.column.is-offset-5-desktop{margin-left:41.66667%}.column.is-6-desktop{flex:none;width:50%}.column.is-offset-6-desktop{margin-left:50%}.column.is-7-desktop{flex:none;width:58.33333%}.column.is-offset-7-desktop{margin-left:58.33333%}.column.is-8-desktop{flex:none;width:66.66667%}.column.is-offset-8-desktop{margin-left:66.66667%}.column.is-9-desktop{flex:none;width:75%}.column.is-offset-9-desktop{margin-left:75%}.column.is-10-desktop{flex:none;width:83.33333%}.column.is-offset-10-desktop{margin-left:83.33333%}.column.is-11-desktop{flex:none;width:91.66667%}.column.is-offset-11-desktop{margin-left:91.66667%}.column.is-12-desktop{flex:none;width:100%}.column.is-offset-12-desktop{margin-left:100%}}@media screen and (min-width:1216px){.column.is-narrow-widescreen{flex:none}.column.is-full-widescreen{flex:none;width:100%}.column.is-three-quarters-widescreen{flex:none;width:75%}.column.is-two-thirds-widescreen{flex:none;width:66.6666%}.column.is-half-widescreen{flex:none;width:50%}.column.is-one-third-widescreen{flex:none;width:33.3333%}.column.is-one-quarter-widescreen{flex:none;width:25%}.column.is-one-fifth-widescreen{flex:none;width:20%}.column.is-two-fifths-widescreen{flex:none;width:40%}.column.is-three-fifths-widescreen{flex:none;width:60%}.column.is-four-fifths-widescreen{flex:none;width:80%}.column.is-offset-three-quarters-widescreen{margin-left:75%}.column.is-offset-two-thirds-widescreen{margin-left:66.6666%}.column.is-offset-half-widescreen{margin-left:50%}.column.is-offset-one-third-widescreen{margin-left:33.3333%}.column.is-offset-one-quarter-widescreen{margin-left:25%}.column.is-offset-one-fifth-widescreen{margin-left:20%}.column.is-offset-two-fifths-widescreen{margin-left:40%}.column.is-offset-three-fifths-widescreen{margin-left:60%}.column.is-offset-four-fifths-widescreen{margin-left:80%}.column.is-0-widescreen{flex:none;width:0%}.column.is-offset-0-widescreen{margin-left:0}.column.is-1-widescreen{flex:none;width:8.33333%}.column.is-offset-1-widescreen{margin-left:8.33333%}.column.is-2-widescreen{flex:none;width:16.66667%}.column.is-offset-2-widescreen{margin-left:16.66667%}.column.is-3-widescreen{flex:none;width:25%}.column.is-offset-3-widescreen{margin-left:25%}.column.is-4-widescreen{flex:none;width:33.33333%}.column.is-offset-4-widescreen{margin-left:33.33333%}.column.is-5-widescreen{flex:none;width:41.66667%}.column.is-offset-5-widescreen{margin-left:41.66667%}.column.is-6-widescreen{flex:none;width:50%}.column.is-offset-6-widescreen{margin-left:50%}.column.is-7-widescreen{flex:none;width:58.33333%}.column.is-offset-7-widescreen{margin-left:58.33333%}.column.is-8-widescreen{flex:none;width:66.66667%}.column.is-offset-8-widescreen{margin-left:66.66667%}.column.is-9-widescreen{flex:none;width:75%}.column.is-offset-9-widescreen{margin-left:75%}.column.is-10-widescreen{flex:none;width:83.33333%}.column.is-offset-10-widescreen{margin-left:83.33333%}.column.is-11-widescreen{flex:none;width:91.66667%}.column.is-offset-11-widescreen{margin-left:91.66667%}.column.is-12-widescreen{flex:none;width:100%}.column.is-offset-12-widescreen{margin-left:100%}}@media screen and (min-width:1408px){.column.is-narrow-fullhd{flex:none}.column.is-full-fullhd{flex:none;width:100%}.column.is-three-quarters-fullhd{flex:none;width:75%}.column.is-two-thirds-fullhd{flex:none;width:66.6666%}.column.is-half-fullhd{flex:none;width:50%}.column.is-one-third-fullhd{flex:none;width:33.3333%}.column.is-one-quarter-fullhd{flex:none;width:25%}.column.is-one-fifth-fullhd{flex:none;width:20%}.column.is-two-fifths-fullhd{flex:none;width:40%}.column.is-three-fifths-fullhd{flex:none;width:60%}.column.is-four-fifths-fullhd{flex:none;width:80%}.column.is-offset-three-quarters-fullhd{margin-left:75%}.column.is-offset-two-thirds-fullhd{margin-left:66.6666%}.column.is-offset-half-fullhd{margin-left:50%}.column.is-offset-one-third-fullhd{margin-left:33.3333%}.column.is-offset-one-quarter-fullhd{margin-left:25%}.column.is-offset-one-fifth-fullhd{margin-left:20%}.column.is-offset-two-fifths-fullhd{margin-left:40%}.column.is-offset-three-fifths-fullhd{margin-left:60%}.column.is-offset-four-fifths-fullhd{margin-left:80%}.column.is-0-fullhd{flex:none;width:0%}.column.is-offset-0-fullhd{margin-left:0}.column.is-1-fullhd{flex:none;width:8.33333%}.column.is-offset-1-fullhd{margin-left:8.33333%}.column.is-2-fullhd{flex:none;width:16.66667%}.column.is-offset-2-fullhd{margin-left:16.66667%}.column.is-3-fullhd{flex:none;width:25%}.column.is-offset-3-fullhd{margin-left:25%}.column.is-4-fullhd{flex:none;width:33.33333%}.column.is-offset-4-fullhd{margin-left:33.33333%}.column.is-5-fullhd{flex:none;width:41.66667%}.column.is-offset-5-fullhd{margin-left:41.66667%}.column.is-6-fullhd{flex:none;width:50%}.column.is-offset-6-fullhd{margin-left:50%}.column.is-7-fullhd{flex:none;width:58.33333%}.column.is-offset-7-fullhd{margin-left:58.33333%}.column.is-8-fullhd{flex:none;width:66.66667%}.column.is-offset-8-fullhd{margin-left:66.66667%}.column.is-9-fullhd{flex:none;width:75%}.column.is-offset-9-fullhd{margin-left:75%}.column.is-10-fullhd{flex:none;width:83.33333%}.column.is-offset-10-fullhd{margin-left:83.33333%}.column.is-11-fullhd{flex:none;width:91.66667%}.column.is-offset-11-fullhd{margin-left:91.66667%}.column.is-12-fullhd{flex:none;width:100%}.column.is-offset-12-fullhd{margin-left:100%}}.columns{margin-left:-.75rem;margin-right:-.75rem;margin-top:-.75rem}.columns:last-child{margin-bottom:-.75rem}.columns:not(:last-child){margin-bottom:calc(1.5rem - .75rem)}.columns.is-centered{justify-content:center}.columns.is-gapless{margin-left:0;margin-right:0;margin-top:0}.columns.is-gapless>.column{margin:0;padding:0!important}.columns.is-gapless:not(:last-child){margin-bottom:1.5rem}.columns.is-gapless:last-child{margin-bottom:0}.columns.is-mobile{display:flex}.columns.is-multiline{flex-wrap:wrap}.columns.is-vcentered{align-items:center}@media screen and (min-width:769px),print{.columns:not(.is-desktop){display:flex}}@media screen and (min-width:1024px){.columns.is-desktop{display:flex}}.columns.is-variable{--columnGap:0.75rem;margin-left:calc(-1 * var(--columnGap));margin-right:calc(-1 * var(--columnGap))}.columns.is-variable .column{padding-left:var(--columnGap);padding-right:var(--columnGap)}.columns.is-variable.is-0{--columnGap:0rem}@media screen and (max-width:768px){.columns.is-variable.is-0-mobile{--columnGap:0rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-0-tablet{--columnGap:0rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-0-tablet-only{--columnGap:0rem}}@media screen and (max-width:1023px){.columns.is-variable.is-0-touch{--columnGap:0rem}}@media screen and (min-width:1024px){.columns.is-variable.is-0-desktop{--columnGap:0rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-0-desktop-only{--columnGap:0rem}}@media screen and (min-width:1216px){.columns.is-variable.is-0-widescreen{--columnGap:0rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-0-widescreen-only{--columnGap:0rem}}@media screen and (min-width:1408px){.columns.is-variable.is-0-fullhd{--columnGap:0rem}}.columns.is-variable.is-1{--columnGap:0.25rem}@media screen and (max-width:768px){.columns.is-variable.is-1-mobile{--columnGap:0.25rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-1-tablet{--columnGap:0.25rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-1-tablet-only{--columnGap:0.25rem}}@media screen and (max-width:1023px){.columns.is-variable.is-1-touch{--columnGap:0.25rem}}@media screen and (min-width:1024px){.columns.is-variable.is-1-desktop{--columnGap:0.25rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-1-desktop-only{--columnGap:0.25rem}}@media screen and (min-width:1216px){.columns.is-variable.is-1-widescreen{--columnGap:0.25rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-1-widescreen-only{--columnGap:0.25rem}}@media screen and (min-width:1408px){.columns.is-variable.is-1-fullhd{--columnGap:0.25rem}}.columns.is-variable.is-2{--columnGap:0.5rem}@media screen and (max-width:768px){.columns.is-variable.is-2-mobile{--columnGap:0.5rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-2-tablet{--columnGap:0.5rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-2-tablet-only{--columnGap:0.5rem}}@media screen and (max-width:1023px){.columns.is-variable.is-2-touch{--columnGap:0.5rem}}@media screen and (min-width:1024px){.columns.is-variable.is-2-desktop{--columnGap:0.5rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-2-desktop-only{--columnGap:0.5rem}}@media screen and (min-width:1216px){.columns.is-variable.is-2-widescreen{--columnGap:0.5rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-2-widescreen-only{--columnGap:0.5rem}}@media screen and (min-width:1408px){.columns.is-variable.is-2-fullhd{--columnGap:0.5rem}}.columns.is-variable.is-3{--columnGap:0.75rem}@media screen and (max-width:768px){.columns.is-variable.is-3-mobile{--columnGap:0.75rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-3-tablet{--columnGap:0.75rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-3-tablet-only{--columnGap:0.75rem}}@media screen and (max-width:1023px){.columns.is-variable.is-3-touch{--columnGap:0.75rem}}@media screen and (min-width:1024px){.columns.is-variable.is-3-desktop{--columnGap:0.75rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-3-desktop-only{--columnGap:0.75rem}}@media screen and (min-width:1216px){.columns.is-variable.is-3-widescreen{--columnGap:0.75rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-3-widescreen-only{--columnGap:0.75rem}}@media screen and (min-width:1408px){.columns.is-variable.is-3-fullhd{--columnGap:0.75rem}}.columns.is-variable.is-4{--columnGap:1rem}@media screen and (max-width:768px){.columns.is-variable.is-4-mobile{--columnGap:1rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-4-tablet{--columnGap:1rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-4-tablet-only{--columnGap:1rem}}@media screen and (max-width:1023px){.columns.is-variable.is-4-touch{--columnGap:1rem}}@media screen and (min-width:1024px){.columns.is-variable.is-4-desktop{--columnGap:1rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-4-desktop-only{--columnGap:1rem}}@media screen and (min-width:1216px){.columns.is-variable.is-4-widescreen{--columnGap:1rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-4-widescreen-only{--columnGap:1rem}}@media screen and (min-width:1408px){.columns.is-variable.is-4-fullhd{--columnGap:1rem}}.columns.is-variable.is-5{--columnGap:1.25rem}@media screen and (max-width:768px){.columns.is-variable.is-5-mobile{--columnGap:1.25rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-5-tablet{--columnGap:1.25rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-5-tablet-only{--columnGap:1.25rem}}@media screen and (max-width:1023px){.columns.is-variable.is-5-touch{--columnGap:1.25rem}}@media screen and (min-width:1024px){.columns.is-variable.is-5-desktop{--columnGap:1.25rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-5-desktop-only{--columnGap:1.25rem}}@media screen and (min-width:1216px){.columns.is-variable.is-5-widescreen{--columnGap:1.25rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-5-widescreen-only{--columnGap:1.25rem}}@media screen and (min-width:1408px){.columns.is-variable.is-5-fullhd{--columnGap:1.25rem}}.columns.is-variable.is-6{--columnGap:1.5rem}@media screen and (max-width:768px){.columns.is-variable.is-6-mobile{--columnGap:1.5rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-6-tablet{--columnGap:1.5rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-6-tablet-only{--columnGap:1.5rem}}@media screen and (max-width:1023px){.columns.is-variable.is-6-touch{--columnGap:1.5rem}}@media screen and (min-width:1024px){.columns.is-variable.is-6-desktop{--columnGap:1.5rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-6-desktop-only{--columnGap:1.5rem}}@media screen and (min-width:1216px){.columns.is-variable.is-6-widescreen{--columnGap:1.5rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-6-widescreen-only{--columnGap:1.5rem}}@media screen and (min-width:1408px){.columns.is-variable.is-6-fullhd{--columnGap:1.5rem}}.columns.is-variable.is-7{--columnGap:1.75rem}@media screen and (max-width:768px){.columns.is-variable.is-7-mobile{--columnGap:1.75rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-7-tablet{--columnGap:1.75rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-7-tablet-only{--columnGap:1.75rem}}@media screen and (max-width:1023px){.columns.is-variable.is-7-touch{--columnGap:1.75rem}}@media screen and (min-width:1024px){.columns.is-variable.is-7-desktop{--columnGap:1.75rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-7-desktop-only{--columnGap:1.75rem}}@media screen and (min-width:1216px){.columns.is-variable.is-7-widescreen{--columnGap:1.75rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-7-widescreen-only{--columnGap:1.75rem}}@media screen and (min-width:1408px){.columns.is-variable.is-7-fullhd{--columnGap:1.75rem}}.columns.is-variable.is-8{--columnGap:2rem}@media screen and (max-width:768px){.columns.is-variable.is-8-mobile{--columnGap:2rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-8-tablet{--columnGap:2rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-8-tablet-only{--columnGap:2rem}}@media screen and (max-width:1023px){.columns.is-variable.is-8-touch{--columnGap:2rem}}@media screen and (min-width:1024px){.columns.is-variable.is-8-desktop{--columnGap:2rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-8-desktop-only{--columnGap:2rem}}@media screen and (min-width:1216px){.columns.is-variable.is-8-widescreen{--columnGap:2rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-8-widescreen-only{--columnGap:2rem}}@media screen and (min-width:1408px){.columns.is-variable.is-8-fullhd{--columnGap:2rem}}.tile{align-items:stretch;display:block;flex-basis:0;flex-grow:1;flex-shrink:1;min-height:-webkit-min-content;min-height:-moz-min-content;min-height:min-content}.tile.is-ancestor{margin-left:-.75rem;margin-right:-.75rem;margin-top:-.75rem}.tile.is-ancestor:last-child{margin-bottom:-.75rem}.tile.is-ancestor:not(:last-child){margin-bottom:.75rem}.tile.is-child{margin:0!important}.tile.is-parent{padding:.75rem}.tile.is-vertical{flex-direction:column}.tile.is-vertical>.tile.is-child:not(:last-child){margin-bottom:1.5rem!important}@media screen and (min-width:769px),print{.tile:not(.is-child){display:flex}.tile.is-1{flex:none;width:8.33333%}.tile.is-2{flex:none;width:16.66667%}.tile.is-3{flex:none;width:25%}.tile.is-4{flex:none;width:33.33333%}.tile.is-5{flex:none;width:41.66667%}.tile.is-6{flex:none;width:50%}.tile.is-7{flex:none;width:58.33333%}.tile.is-8{flex:none;width:66.66667%}.tile.is-9{flex:none;width:75%}.tile.is-10{flex:none;width:83.33333%}.tile.is-11{flex:none;width:91.66667%}.tile.is-12{flex:none;width:100%}}.hero{align-items:stretch;display:flex;flex-direction:column;justify-content:space-between}.hero .navbar{background:0 0}.hero .tabs ul{border-bottom:none}.hero.is-white{background-color:#fff;color:#0a0a0a}.hero.is-white a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-white strong{color:inherit}.hero.is-white .title{color:#0a0a0a}.hero.is-white .subtitle{color:rgba(10,10,10,.9)}.hero.is-white .subtitle a:not(.button),.hero.is-white .subtitle strong{color:#0a0a0a}@media screen and (max-width:1023px){.hero.is-white .navbar-menu{background-color:#fff}}.hero.is-white .navbar-item,.hero.is-white .navbar-link{color:rgba(10,10,10,.7)}.hero.is-white .navbar-link.is-active,.hero.is-white .navbar-link:hover,.hero.is-white a.navbar-item.is-active,.hero.is-white a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.hero.is-white .tabs a{color:#0a0a0a;opacity:.9}.hero.is-white .tabs a:hover{opacity:1}.hero.is-white .tabs li.is-active a{opacity:1}.hero.is-white .tabs.is-boxed a,.hero.is-white .tabs.is-toggle a{color:#0a0a0a}.hero.is-white .tabs.is-boxed a:hover,.hero.is-white .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-white .tabs.is-boxed li.is-active a,.hero.is-white .tabs.is-boxed li.is-active a:hover,.hero.is-white .tabs.is-toggle li.is-active a,.hero.is-white .tabs.is-toggle li.is-active a:hover{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.hero.is-white.is-bold{background-image:linear-gradient(141deg,#e6e6e6 0,#fff 71%,#fff 100%)}@media screen and (max-width:768px){.hero.is-white.is-bold .navbar-menu{background-image:linear-gradient(141deg,#e6e6e6 0,#fff 71%,#fff 100%)}}.hero.is-black{background-color:#0a0a0a;color:#fff}.hero.is-black a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-black strong{color:inherit}.hero.is-black .title{color:#fff}.hero.is-black .subtitle{color:rgba(255,255,255,.9)}.hero.is-black .subtitle a:not(.button),.hero.is-black .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-black .navbar-menu{background-color:#0a0a0a}}.hero.is-black .navbar-item,.hero.is-black .navbar-link{color:rgba(255,255,255,.7)}.hero.is-black .navbar-link.is-active,.hero.is-black .navbar-link:hover,.hero.is-black a.navbar-item.is-active,.hero.is-black a.navbar-item:hover{background-color:#000;color:#fff}.hero.is-black .tabs a{color:#fff;opacity:.9}.hero.is-black .tabs a:hover{opacity:1}.hero.is-black .tabs li.is-active a{opacity:1}.hero.is-black .tabs.is-boxed a,.hero.is-black .tabs.is-toggle a{color:#fff}.hero.is-black .tabs.is-boxed a:hover,.hero.is-black .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-black .tabs.is-boxed li.is-active a,.hero.is-black .tabs.is-boxed li.is-active a:hover,.hero.is-black .tabs.is-toggle li.is-active a,.hero.is-black .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#0a0a0a}.hero.is-black.is-bold{background-image:linear-gradient(141deg,#000 0,#0a0a0a 71%,#181616 100%)}@media screen and (max-width:768px){.hero.is-black.is-bold .navbar-menu{background-image:linear-gradient(141deg,#000 0,#0a0a0a 71%,#181616 100%)}}.hero.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.hero.is-light a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-light strong{color:inherit}.hero.is-light .title{color:rgba(0,0,0,.7)}.hero.is-light .subtitle{color:rgba(0,0,0,.9)}.hero.is-light .subtitle a:not(.button),.hero.is-light .subtitle strong{color:rgba(0,0,0,.7)}@media screen and (max-width:1023px){.hero.is-light .navbar-menu{background-color:#f5f5f5}}.hero.is-light .navbar-item,.hero.is-light .navbar-link{color:rgba(0,0,0,.7)}.hero.is-light .navbar-link.is-active,.hero.is-light .navbar-link:hover,.hero.is-light a.navbar-item.is-active,.hero.is-light a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.hero.is-light .tabs a{color:rgba(0,0,0,.7);opacity:.9}.hero.is-light .tabs a:hover{opacity:1}.hero.is-light .tabs li.is-active a{opacity:1}.hero.is-light .tabs.is-boxed a,.hero.is-light .tabs.is-toggle a{color:rgba(0,0,0,.7)}.hero.is-light .tabs.is-boxed a:hover,.hero.is-light .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-light .tabs.is-boxed li.is-active a,.hero.is-light .tabs.is-boxed li.is-active a:hover,.hero.is-light .tabs.is-toggle li.is-active a,.hero.is-light .tabs.is-toggle li.is-active a:hover{background-color:rgba(0,0,0,.7);border-color:rgba(0,0,0,.7);color:#f5f5f5}.hero.is-light.is-bold{background-image:linear-gradient(141deg,#dfd8d9 0,#f5f5f5 71%,#fff 100%)}@media screen and (max-width:768px){.hero.is-light.is-bold .navbar-menu{background-image:linear-gradient(141deg,#dfd8d9 0,#f5f5f5 71%,#fff 100%)}}.hero.is-dark{background-color:#363636;color:#fff}.hero.is-dark a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-dark strong{color:inherit}.hero.is-dark .title{color:#fff}.hero.is-dark .subtitle{color:rgba(255,255,255,.9)}.hero.is-dark .subtitle a:not(.button),.hero.is-dark .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-dark .navbar-menu{background-color:#363636}}.hero.is-dark .navbar-item,.hero.is-dark .navbar-link{color:rgba(255,255,255,.7)}.hero.is-dark .navbar-link.is-active,.hero.is-dark .navbar-link:hover,.hero.is-dark a.navbar-item.is-active,.hero.is-dark a.navbar-item:hover{background-color:#292929;color:#fff}.hero.is-dark .tabs a{color:#fff;opacity:.9}.hero.is-dark .tabs a:hover{opacity:1}.hero.is-dark .tabs li.is-active a{opacity:1}.hero.is-dark .tabs.is-boxed a,.hero.is-dark .tabs.is-toggle a{color:#fff}.hero.is-dark .tabs.is-boxed a:hover,.hero.is-dark .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-dark .tabs.is-boxed li.is-active a,.hero.is-dark .tabs.is-boxed li.is-active a:hover,.hero.is-dark .tabs.is-toggle li.is-active a,.hero.is-dark .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#363636}.hero.is-dark.is-bold{background-image:linear-gradient(141deg,#1f191a 0,#363636 71%,#46403f 100%)}@media screen and (max-width:768px){.hero.is-dark.is-bold .navbar-menu{background-image:linear-gradient(141deg,#1f191a 0,#363636 71%,#46403f 100%)}}.hero.is-primary{background-color:#00d1b2;color:#fff}.hero.is-primary a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-primary strong{color:inherit}.hero.is-primary .title{color:#fff}.hero.is-primary .subtitle{color:rgba(255,255,255,.9)}.hero.is-primary .subtitle a:not(.button),.hero.is-primary .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-primary .navbar-menu{background-color:#00d1b2}}.hero.is-primary .navbar-item,.hero.is-primary .navbar-link{color:rgba(255,255,255,.7)}.hero.is-primary .navbar-link.is-active,.hero.is-primary .navbar-link:hover,.hero.is-primary a.navbar-item.is-active,.hero.is-primary a.navbar-item:hover{background-color:#00b89c;color:#fff}.hero.is-primary .tabs a{color:#fff;opacity:.9}.hero.is-primary .tabs a:hover{opacity:1}.hero.is-primary .tabs li.is-active a{opacity:1}.hero.is-primary .tabs.is-boxed a,.hero.is-primary .tabs.is-toggle a{color:#fff}.hero.is-primary .tabs.is-boxed a:hover,.hero.is-primary .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-primary .tabs.is-boxed li.is-active a,.hero.is-primary .tabs.is-boxed li.is-active a:hover,.hero.is-primary .tabs.is-toggle li.is-active a,.hero.is-primary .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#00d1b2}.hero.is-primary.is-bold{background-image:linear-gradient(141deg,#009e6c 0,#00d1b2 71%,#00e7eb 100%)}@media screen and (max-width:768px){.hero.is-primary.is-bold .navbar-menu{background-image:linear-gradient(141deg,#009e6c 0,#00d1b2 71%,#00e7eb 100%)}}.hero.is-link{background-color:#3273dc;color:#fff}.hero.is-link a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-link strong{color:inherit}.hero.is-link .title{color:#fff}.hero.is-link .subtitle{color:rgba(255,255,255,.9)}.hero.is-link .subtitle a:not(.button),.hero.is-link .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-link .navbar-menu{background-color:#3273dc}}.hero.is-link .navbar-item,.hero.is-link .navbar-link{color:rgba(255,255,255,.7)}.hero.is-link .navbar-link.is-active,.hero.is-link .navbar-link:hover,.hero.is-link a.navbar-item.is-active,.hero.is-link a.navbar-item:hover{background-color:#2366d1;color:#fff}.hero.is-link .tabs a{color:#fff;opacity:.9}.hero.is-link .tabs a:hover{opacity:1}.hero.is-link .tabs li.is-active a{opacity:1}.hero.is-link .tabs.is-boxed a,.hero.is-link .tabs.is-toggle a{color:#fff}.hero.is-link .tabs.is-boxed a:hover,.hero.is-link .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-link .tabs.is-boxed li.is-active a,.hero.is-link .tabs.is-boxed li.is-active a:hover,.hero.is-link .tabs.is-toggle li.is-active a,.hero.is-link .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#3273dc}.hero.is-link.is-bold{background-image:linear-gradient(141deg,#1577c6 0,#3273dc 71%,#4366e5 100%)}@media screen and (max-width:768px){.hero.is-link.is-bold .navbar-menu{background-image:linear-gradient(141deg,#1577c6 0,#3273dc 71%,#4366e5 100%)}}.hero.is-info{background-color:#3298dc;color:#fff}.hero.is-info a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-info strong{color:inherit}.hero.is-info .title{color:#fff}.hero.is-info .subtitle{color:rgba(255,255,255,.9)}.hero.is-info .subtitle a:not(.button),.hero.is-info .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-info .navbar-menu{background-color:#3298dc}}.hero.is-info .navbar-item,.hero.is-info .navbar-link{color:rgba(255,255,255,.7)}.hero.is-info .navbar-link.is-active,.hero.is-info .navbar-link:hover,.hero.is-info a.navbar-item.is-active,.hero.is-info a.navbar-item:hover{background-color:#238cd1;color:#fff}.hero.is-info .tabs a{color:#fff;opacity:.9}.hero.is-info .tabs a:hover{opacity:1}.hero.is-info .tabs li.is-active a{opacity:1}.hero.is-info .tabs.is-boxed a,.hero.is-info .tabs.is-toggle a{color:#fff}.hero.is-info .tabs.is-boxed a:hover,.hero.is-info .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-info .tabs.is-boxed li.is-active a,.hero.is-info .tabs.is-boxed li.is-active a:hover,.hero.is-info .tabs.is-toggle li.is-active a,.hero.is-info .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#3298dc}.hero.is-info.is-bold{background-image:linear-gradient(141deg,#159dc6 0,#3298dc 71%,#4389e5 100%)}@media screen and (max-width:768px){.hero.is-info.is-bold .navbar-menu{background-image:linear-gradient(141deg,#159dc6 0,#3298dc 71%,#4389e5 100%)}}.hero.is-success{background-color:#48c774;color:#fff}.hero.is-success a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-success strong{color:inherit}.hero.is-success .title{color:#fff}.hero.is-success .subtitle{color:rgba(255,255,255,.9)}.hero.is-success .subtitle a:not(.button),.hero.is-success .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-success .navbar-menu{background-color:#48c774}}.hero.is-success .navbar-item,.hero.is-success .navbar-link{color:rgba(255,255,255,.7)}.hero.is-success .navbar-link.is-active,.hero.is-success .navbar-link:hover,.hero.is-success a.navbar-item.is-active,.hero.is-success a.navbar-item:hover{background-color:#3abb67;color:#fff}.hero.is-success .tabs a{color:#fff;opacity:.9}.hero.is-success .tabs a:hover{opacity:1}.hero.is-success .tabs li.is-active a{opacity:1}.hero.is-success .tabs.is-boxed a,.hero.is-success .tabs.is-toggle a{color:#fff}.hero.is-success .tabs.is-boxed a:hover,.hero.is-success .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-success .tabs.is-boxed li.is-active a,.hero.is-success .tabs.is-boxed li.is-active a:hover,.hero.is-success .tabs.is-toggle li.is-active a,.hero.is-success .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#48c774}.hero.is-success.is-bold{background-image:linear-gradient(141deg,#29b342 0,#48c774 71%,#56d296 100%)}@media screen and (max-width:768px){.hero.is-success.is-bold .navbar-menu{background-image:linear-gradient(141deg,#29b342 0,#48c774 71%,#56d296 100%)}}.hero.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.hero.is-warning a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-warning strong{color:inherit}.hero.is-warning .title{color:rgba(0,0,0,.7)}.hero.is-warning .subtitle{color:rgba(0,0,0,.9)}.hero.is-warning .subtitle a:not(.button),.hero.is-warning .subtitle strong{color:rgba(0,0,0,.7)}@media screen and (max-width:1023px){.hero.is-warning .navbar-menu{background-color:#ffdd57}}.hero.is-warning .navbar-item,.hero.is-warning .navbar-link{color:rgba(0,0,0,.7)}.hero.is-warning .navbar-link.is-active,.hero.is-warning .navbar-link:hover,.hero.is-warning a.navbar-item.is-active,.hero.is-warning a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.hero.is-warning .tabs a{color:rgba(0,0,0,.7);opacity:.9}.hero.is-warning .tabs a:hover{opacity:1}.hero.is-warning .tabs li.is-active a{opacity:1}.hero.is-warning .tabs.is-boxed a,.hero.is-warning .tabs.is-toggle a{color:rgba(0,0,0,.7)}.hero.is-warning .tabs.is-boxed a:hover,.hero.is-warning .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-warning .tabs.is-boxed li.is-active a,.hero.is-warning .tabs.is-boxed li.is-active a:hover,.hero.is-warning .tabs.is-toggle li.is-active a,.hero.is-warning .tabs.is-toggle li.is-active a:hover{background-color:rgba(0,0,0,.7);border-color:rgba(0,0,0,.7);color:#ffdd57}.hero.is-warning.is-bold{background-image:linear-gradient(141deg,#ffaf24 0,#ffdd57 71%,#fffa70 100%)}@media screen and (max-width:768px){.hero.is-warning.is-bold .navbar-menu{background-image:linear-gradient(141deg,#ffaf24 0,#ffdd57 71%,#fffa70 100%)}}.hero.is-danger{background-color:#f14668;color:#fff}.hero.is-danger a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-danger strong{color:inherit}.hero.is-danger .title{color:#fff}.hero.is-danger .subtitle{color:rgba(255,255,255,.9)}.hero.is-danger .subtitle a:not(.button),.hero.is-danger .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-danger .navbar-menu{background-color:#f14668}}.hero.is-danger .navbar-item,.hero.is-danger .navbar-link{color:rgba(255,255,255,.7)}.hero.is-danger .navbar-link.is-active,.hero.is-danger .navbar-link:hover,.hero.is-danger a.navbar-item.is-active,.hero.is-danger a.navbar-item:hover{background-color:#ef2e55;color:#fff}.hero.is-danger .tabs a{color:#fff;opacity:.9}.hero.is-danger .tabs a:hover{opacity:1}.hero.is-danger .tabs li.is-active a{opacity:1}.hero.is-danger .tabs.is-boxed a,.hero.is-danger .tabs.is-toggle a{color:#fff}.hero.is-danger .tabs.is-boxed a:hover,.hero.is-danger .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-danger .tabs.is-boxed li.is-active a,.hero.is-danger .tabs.is-boxed li.is-active a:hover,.hero.is-danger .tabs.is-toggle li.is-active a,.hero.is-danger .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#f14668}.hero.is-danger.is-bold{background-image:linear-gradient(141deg,#fa0a62 0,#f14668 71%,#f7595f 100%)}@media screen and (max-width:768px){.hero.is-danger.is-bold .navbar-menu{background-image:linear-gradient(141deg,#fa0a62 0,#f14668 71%,#f7595f 100%)}}.hero.is-small .hero-body{padding-bottom:1.5rem;padding-top:1.5rem}@media screen and (min-width:769px),print{.hero.is-medium .hero-body{padding-bottom:9rem;padding-top:9rem}}@media screen and (min-width:769px),print{.hero.is-large .hero-body{padding-bottom:18rem;padding-top:18rem}}.hero.is-fullheight .hero-body,.hero.is-fullheight-with-navbar .hero-body,.hero.is-halfheight .hero-body{align-items:center;display:flex}.hero.is-fullheight .hero-body>.container,.hero.is-fullheight-with-navbar .hero-body>.container,.hero.is-halfheight .hero-body>.container{flex-grow:1;flex-shrink:1}.hero.is-halfheight{min-height:50vh}.hero.is-fullheight{min-height:100vh}.hero-video{overflow:hidden}.hero-video video{left:50%;min-height:100%;min-width:100%;position:absolute;top:50%;transform:translate3d(-50%,-50%,0)}.hero-video.is-transparent{opacity:.3}@media screen and (max-width:768px){.hero-video{display:none}}.hero-buttons{margin-top:1.5rem}@media screen and (max-width:768px){.hero-buttons .button{display:flex}.hero-buttons .button:not(:last-child){margin-bottom:.75rem}}@media screen and (min-width:769px),print{.hero-buttons{display:flex;justify-content:center}.hero-buttons .button:not(:last-child){margin-right:1.5rem}}.hero-foot,.hero-head{flex-grow:0;flex-shrink:0}.hero-body{flex-grow:1;flex-shrink:0;padding:3rem 1.5rem}.section{padding:3rem 1.5rem}@media screen and (min-width:1024px){.section.is-medium{padding:9rem 1.5rem}.section.is-large{padding:18rem 1.5rem}}.footer{background-color:#fafafa;padding:3rem 1.5rem 6rem} \ No newline at end of file diff --git a/meilisearch-http/public/interface.html b/meilisearch-http/public/interface.html deleted file mode 100644 index ca3005392..000000000 --- a/meilisearch-http/public/interface.html +++ /dev/null @@ -1,333 +0,0 @@ - - - - - - - MeiliSearch - - - - -
- -
-
-
-

- Welcome to MeiliSearch -

-

- This dashboard will help you check the search results with ease. -

-
-
-
-
- - -
- -
-

At least a private API key is required for the dashboard to access the indexes list.

-
-
-
-
-
- -
-
- - - -
-
- -
-
-
-
-
-
-

Documents

-

0

-
-
-

Time Spent

-

N/A

-
-
-
-
-
-
-
- -
-
-
    - -
-
-
- - - - diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 44166d1ff..fd5cf6786 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -9,7 +9,6 @@ pub mod routes; pub use self::data::Data; pub use option::Opt; - #[macro_export] macro_rules! create_app { ($data:expr, $enable_frontend:expr) => {{ @@ -52,7 +51,6 @@ macro_rules! create_app { #[cfg(feature = "mini-dashboard")] let app = if $enable_frontend { let generated = dashboard::generate(); - let keys = generated.keys().collect::>(); let service = ResourceFiles::new("/", generated); app.service(service) } else { diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 1163c4f24..b16f3c0e1 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -72,20 +72,14 @@ async fn main() -> Result<(), MainError> { print_launch_resume(&opt, &data); - let enable_frontend = opt.env != "production"; - - run_http(data, opt, enable_frontend).await?; + run_http(data, opt).await?; Ok(()) } -#[allow(unused_variables)] -async fn run_http( - data: Data, - opt: Opt, - enable_frontend: bool, -) -> Result<(), Box> { - let http_server = HttpServer::new(move || create_app!(&data, enable_frontend)) +async fn run_http(data: Data, opt: Opt) -> Result<(), Box> { + let _enable_dashboard = &opt.env == "development"; + let http_server = HttpServer::new(move || create_app!(&data, _enable_dashboard)) // Disable signals allows the server to terminate immediately when a user enter CTRL-C .disable_signals(); @@ -103,11 +97,11 @@ async fn run_http( pub fn print_launch_resume(opt: &Opt, data: &Data) { let commit_sha = match option_env!("COMMIT_SHA") { Some("") | None => env!("VERGEN_SHA"), - Some(commit_sha) => commit_sha + Some(commit_sha) => commit_sha, }; let commit_date = match option_env!("COMMIT_DATE") { Some("") | None => env!("VERGEN_COMMIT_DATE"), - Some(commit_date) => commit_date + Some(commit_date) => commit_date, }; let ascii_name = r#" diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index f1c559705..aaf13613a 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -28,14 +28,6 @@ impl IndexUpdateResponse { } } -/// Return the dashboard, should not be used in production. See [running] -#[get("/")] -pub async fn load_html() -> HttpResponse { - HttpResponse::Ok() - .content_type("text/html; charset=utf-8") - .body(include_str!("../../public/interface.html").to_string()) -} - /// Always return a 200 with: /// ```json /// { @@ -46,10 +38,3 @@ pub async fn load_html() -> HttpResponse { pub async fn running() -> HttpResponse { HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" })) } - -#[get("/bulma.min.css")] -pub async fn load_css() -> HttpResponse { - HttpResponse::Ok() - .content_type("text/css; charset=utf-8") - .body(include_str!("../../public/bulma.min.css").to_string()) -} From 77481d7c768e4f2385aae1169c8136ba01bb2bc3 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 26 Apr 2021 18:19:54 +0200 Subject: [PATCH 280/522] update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e1f56a99c..04bae1630 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ /target -meilisearch-core/target **/*.csv **/*.json_lines **/*.rs.bk /*.mdb /query-history.txt /data.ms +/meilisearch-http/mini-dashboard +/meilisearch-http/.mini-dashboard.sha1 From f3b6bf55a6b804b1864ad4ef09e036fd5c8091bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 26 Apr 2021 19:05:16 +0200 Subject: [PATCH 281/522] Update version for the next release (alpha4) --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 879e43de9..13f023140 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1742,7 +1742,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.21.0-alpha.3" +version = "0.21.0-alpha.4" dependencies = [ "actix-cors", "actix-http", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index a5d5bc14d..c5aef1a56 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "MeiliSearch HTTP server" edition = "2018" license = "MIT" name = "meilisearch-http" -version = "0.21.0-alpha.3" +version = "0.21.0-alpha.4" [[bin]] name = "meilisearch" From 5014f74649680d041a286e2b6775e04b0d5f0ea1 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 27 Apr 2021 09:32:17 +0200 Subject: [PATCH 282/522] put mini-dashboard in out-dir --- .gitignore | 2 -- meilisearch-http/build.rs | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 04bae1630..3ae73d6d8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,3 @@ /*.mdb /query-history.txt /data.ms -/meilisearch-http/mini-dashboard -/meilisearch-http/.mini-dashboard.sha1 diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index 8cc743543..5dbde1477 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -28,9 +28,10 @@ mod mini_dashboard { pub fn setup_mini_dashboard() -> anyhow::Result<()> { let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let cargo_toml = cargo_manifest_dir.join("Cargo.toml"); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - let sha1_path = cargo_manifest_dir.join(".mini-dashboard.sha1"); - let dashboard_dir = cargo_manifest_dir.join("mini-dashboard"); + let sha1_path = out_dir.join(".mini-dashboard.sha1"); + let dashboard_dir = out_dir.join("mini-dashboard"); let manifest = Manifest::from_path(cargo_toml).unwrap(); From f80ea24d2b581bd6c1a922992072f2f0873f8a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 22 Mar 2021 19:17:18 +0100 Subject: [PATCH 283/522] Add tests on every platform and fix clippy errors --- .github/workflows/rust.yml | 46 ++++++++++++++++--- bors.toml | 7 ++- meilisearch-http/src/index/search.rs | 2 +- .../src/index_controller/index_actor/actor.rs | 2 +- .../update_actor/update_store.rs | 20 ++++---- meilisearch-http/src/routes/stats.rs | 3 +- meilisearch-http/tests/common/index.rs | 2 +- meilisearch-http/tests/common/server.rs | 2 +- meilisearch-http/tests/common/service.rs | 20 ++++---- meilisearch-http/tests/index/get_index.rs | 6 +-- meilisearch-http/tests/snapshot/mod.rs | 2 +- 11 files changed, 73 insertions(+), 39 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 230339c49..3ea29b253 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -13,12 +13,44 @@ env: jobs: tests: - - runs-on: ubuntu-latest - + name: Tests on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04, macos-latest] steps: - uses: actions/checkout@v2 - - name: Build - run: cargo build --verbose - - name: Run tests - run: cargo test --verbose + - name: Run cargo test + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --release + + # We don't run test on Windows since we get the following error: There is not enough space on the disk. + build-on-windows: + name: Build on windows-latest + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: Run cargo test + uses: actions-rs/cargo@v1 + with: + command: build + + clippy: + name: Run Clippy + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + components: clippy + - name: Run cargo clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all-targets -- --deny warnings diff --git a/bors.toml b/bors.toml index dd19e1f5b..2e2da90ac 100644 --- a/bors.toml +++ b/bors.toml @@ -1,3 +1,8 @@ -status = ['tests'] +status = [ + 'Tests on ubuntu-18.04', + 'Tests on macos-latest', + 'Build on windows-latest', + 'Run Clippy' +] # 3 hours timeout timeout-sec = 10800 diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 1cccfcd58..0ff6c1bc3 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -162,7 +162,7 @@ impl Index { ); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let document = make_document(&all_attributes, &fields_ids_map, obkv.clone())?; + let document = make_document(&all_attributes, &fields_ids_map, obkv)?; let formatted = compute_formatted( &fields_ids_map, obkv, diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index c54b2edd2..9cca6557b 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -197,7 +197,7 @@ impl IndexActor { .map_err(|e| IndexError::Error(e.into())) } - *self.processing.write().await = Some(meta.index_uuid().clone()); + *self.processing.write().await = Some(*meta.index_uuid()); let result = get_result(self, meta, data).await; *self.processing.write().await = None; diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 3d6c4e396..5cf00da34 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -13,16 +13,16 @@ use uuid::Uuid; use crate::helpers::EnvSizer; use crate::index_controller::updates::*; -type BEU64 = heed::zerocopy::U64; +type Beu64 = heed::zerocopy::U64; #[derive(Clone)] pub struct UpdateStore { pub env: Env, - pending_meta: Database, SerdeJson>>, - pending: Database, SerdeJson>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, + pending_meta: Database, SerdeJson>>, + pending: Database, SerdeJson>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, processing: Arc>>>, notification_sender: mpsc::Sender<()>, /// A lock on the update loop. This is meant to prevent a snapshot to occur while an update is @@ -176,7 +176,7 @@ where // asking for the id and registering it so other update registering // will be forced to wait for a new write txn. let update_id = self.new_update_id(&wtxn)?; - let update_key = BEU64::new(update_id); + let update_key = Beu64::new(update_id); let meta = Enqueued::new(meta, update_id, index_uuid); self.pending_meta.put(&mut wtxn, &update_key, &meta)?; @@ -295,7 +295,7 @@ where /// Returns the update associated meta or `None` if the update doesn't exist. pub fn meta(&self, update_id: u64) -> heed::Result>> { let rtxn = self.env.read_txn()?; - let key = BEU64::new(update_id); + let key = Beu64::new(update_id); if let Some(ref meta) = *self.processing.read() { if meta.id() == update_id { @@ -331,7 +331,7 @@ where #[allow(dead_code)] pub fn abort_update(&self, update_id: u64) -> heed::Result>> { let mut wtxn = self.env.write_txn()?; - let key = BEU64::new(update_id); + let key = Beu64::new(update_id); // We cannot abort an update that is currently being processed. if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { @@ -369,7 +369,7 @@ where } for (id, aborted) in &aborted_updates { - let key = BEU64::new(*id); + let key = Beu64::new(*id); self.aborted_meta.put(&mut wtxn, &key, &aborted)?; self.pending_meta.delete(&mut wtxn, &key)?; self.pending.delete(&mut wtxn, &key)?; diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index 988e6cf40..226b62fcd 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,5 +1,4 @@ use std::collections::BTreeMap; -use std::iter::FromIterator; use actix_web::get; use actix_web::web; @@ -33,7 +32,7 @@ impl From for IndexStatsResponse { Self { number_of_documents: stats.number_of_documents, is_indexing: stats.is_indexing, - fields_distribution: BTreeMap::from_iter(stats.fields_distribution.into_iter()), + fields_distribution: stats.fields_distribution.into_iter().collect(), } } } diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index aaf108cf6..86aa80c3d 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -47,7 +47,7 @@ impl Index<'_> { update_id as u64 } - pub async fn create<'a>(&'a self, primary_key: Option<&str>) -> (Value, StatusCode) { + pub async fn create(& self, primary_key: Option<&str>) -> (Value, StatusCode) { let body = json!({ "uid": self.uid, "primaryKey": primary_key, diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index e814ab3ed..100722ec4 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -44,7 +44,7 @@ impl Server { } /// Returns a view to an index. There is no guarantee that the index exists. - pub fn index<'a>(&'a self, uid: impl AsRef) -> Index<'a> { + pub fn index(& self, uid: impl AsRef) -> Index<'_> { Index { uid: encode(uid.as_ref()), service: &self.service, diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index b9bbffc05..08db5b9dc 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -8,13 +8,13 @@ pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let mut app = test::init_service(create_app!(&self.0, true)).await; + let app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) .set_json(&body) .to_request(); - let res = test::call_service(&mut app, req).await; + let res = test::call_service(&app, req).await; let status_code = res.status(); let body = test::read_body(res).await; @@ -28,14 +28,14 @@ impl Service { url: impl AsRef, body: impl AsRef, ) -> (Value, StatusCode) { - let mut app = test::init_service(create_app!(&self.0, true)).await; + let app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) .insert_header(("content-type", "application/json")) .to_request(); - let res = test::call_service(&mut app, req).await; + let res = test::call_service(&app, req).await; let status_code = res.status(); let body = test::read_body(res).await; @@ -44,10 +44,10 @@ impl Service { } pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { - let mut app = test::init_service(create_app!(&self.0, true)).await; + let app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); - let res = test::call_service(&mut app, req).await; + let res = test::call_service(&app, req).await; let status_code = res.status(); let body = test::read_body(res).await; @@ -56,13 +56,13 @@ impl Service { } pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let mut app = test::init_service(create_app!(&self.0, true)).await; + let app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::put() .uri(url.as_ref()) .set_json(&body) .to_request(); - let res = test::call_service(&mut app, req).await; + let res = test::call_service(&app, req).await; let status_code = res.status(); let body = test::read_body(res).await; @@ -71,10 +71,10 @@ impl Service { } pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { - let mut app = test::init_service(create_app!(&self.0, true)).await; + let app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); - let res = test::call_service(&mut app, req).await; + let res = test::call_service(&app, req).await; let status_code = res.status(); let body = test::read_body(res).await; diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index 8671d85cd..ce4d7b792 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -55,10 +55,8 @@ async fn list_multiple_indexes() { assert_eq!(arr.len(), 2); assert!(arr .iter() - .find(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null) - .is_some()); + .any(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null)); assert!(arr .iter() - .find(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key") - .is_some()); + .any(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key")); } diff --git a/meilisearch-http/tests/snapshot/mod.rs b/meilisearch-http/tests/snapshot/mod.rs index 36763a1ab..caed293e6 100644 --- a/meilisearch-http/tests/snapshot/mod.rs +++ b/meilisearch-http/tests/snapshot/mod.rs @@ -35,7 +35,7 @@ async fn perform_snapshot() { let snapshot_path = snapshot_dir .path() .to_owned() - .join(format!("db.snapshot")); + .join("db.snapshot".to_string()); let options = Opt { import_snapshot: Some(snapshot_path), From 1ba46f8f776409d550495c37591a9718a4404451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 27 Apr 2021 12:16:24 +0200 Subject: [PATCH 284/522] Disable clippy rule --- .../update_actor/update_store.rs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 5cf00da34..f1895829b 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -13,16 +13,17 @@ use uuid::Uuid; use crate::helpers::EnvSizer; use crate::index_controller::updates::*; -type Beu64 = heed::zerocopy::U64; +#[allow(clippy::upper_case_acronyms)] +type BEU64 = heed::zerocopy::U64; #[derive(Clone)] pub struct UpdateStore { pub env: Env, - pending_meta: Database, SerdeJson>>, - pending: Database, SerdeJson>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, + pending_meta: Database, SerdeJson>>, + pending: Database, SerdeJson>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, processing: Arc>>>, notification_sender: mpsc::Sender<()>, /// A lock on the update loop. This is meant to prevent a snapshot to occur while an update is @@ -176,7 +177,7 @@ where // asking for the id and registering it so other update registering // will be forced to wait for a new write txn. let update_id = self.new_update_id(&wtxn)?; - let update_key = Beu64::new(update_id); + let update_key = BEU64::new(update_id); let meta = Enqueued::new(meta, update_id, index_uuid); self.pending_meta.put(&mut wtxn, &update_key, &meta)?; @@ -295,7 +296,7 @@ where /// Returns the update associated meta or `None` if the update doesn't exist. pub fn meta(&self, update_id: u64) -> heed::Result>> { let rtxn = self.env.read_txn()?; - let key = Beu64::new(update_id); + let key = BEU64::new(update_id); if let Some(ref meta) = *self.processing.read() { if meta.id() == update_id { @@ -331,7 +332,7 @@ where #[allow(dead_code)] pub fn abort_update(&self, update_id: u64) -> heed::Result>> { let mut wtxn = self.env.write_txn()?; - let key = Beu64::new(update_id); + let key = BEU64::new(update_id); // We cannot abort an update that is currently being processed. if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { @@ -369,7 +370,7 @@ where } for (id, aborted) in &aborted_updates { - let key = Beu64::new(*id); + let key = BEU64::new(*id); self.aborted_meta.put(&mut wtxn, &key, &aborted)?; self.pending_meta.delete(&mut wtxn, &key)?; self.pending.delete(&mut wtxn, &key)?; From 0c41adf868639e4970674b28e152381b81464178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 27 Apr 2021 12:36:36 +0200 Subject: [PATCH 285/522] Update CI --- .github/workflows/rust.yml | 18 ++++++++++++++---- bors.toml | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3ea29b253..bdbf50a95 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,6 +21,11 @@ jobs: os: [ubuntu-18.04, macos-latest] steps: - uses: actions/checkout@v2 + - name: Run cargo check without any default features + uses: actions-rs/cargo@v1 + with: + command: check + args: --no-default-features - name: Run cargo test uses: actions-rs/cargo@v1 with: @@ -28,15 +33,20 @@ jobs: args: --locked --release # We don't run test on Windows since we get the following error: There is not enough space on the disk. - build-on-windows: - name: Build on windows-latest + check-on-windows: + name: Cargo check on Windows runs-on: windows-latest steps: - uses: actions/checkout@v2 - - name: Run cargo test + - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: - command: build + command: check + args: --no-default-features + - name: Run cargo check with all default features + uses: actions-rs/cargo@v1 + with: + command: check clippy: name: Run Clippy diff --git a/bors.toml b/bors.toml index 2e2da90ac..45133f836 100644 --- a/bors.toml +++ b/bors.toml @@ -1,7 +1,7 @@ status = [ 'Tests on ubuntu-18.04', 'Tests on macos-latest', - 'Build on windows-latest', + 'Cargo check on Windows', 'Run Clippy' ] # 3 hours timeout From 5a38f13cae38f148ef5a9080fd8fccc732199908 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 12 Apr 2021 16:59:16 +0200 Subject: [PATCH 286/522] multi_index udpate store --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 1 + .../update_actor/update_store.rs | 257 +++++++++++++----- .../src/index_controller/updates.rs | 4 +- 4 files changed, 196 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13f023140..25acf9bda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1756,6 +1756,7 @@ dependencies = [ "async-stream", "async-trait", "byte-unit", + "bytemuck", "bytes 0.6.0", "cargo_toml", "chrono", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index c5aef1a56..c1eb6b8dc 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -32,6 +32,7 @@ async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } async-stream = "0.3.0" async-trait = "0.1.42" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } +bytemuck = "1.5.1" bytes = "0.6.0" chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.0" diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index f1895829b..efd827619 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -1,10 +1,13 @@ -use std::fs::File; -use std::fs::{copy, create_dir_all, remove_file}; +use std::borrow::Cow; +use std::convert::TryInto; +use std::fs::{copy, create_dir_all, remove_file, File}; +use std::mem::size_of; use std::path::{Path, PathBuf}; use std::sync::Arc; -use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; -use heed::{CompactionOption, Database, Env, EnvOpenOptions}; +use bytemuck::{Pod, Zeroable}; +use heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; +use heed::{BytesDecode, BytesEncode, CompactionOption, Database, Env, EnvOpenOptions}; use parking_lot::{Mutex, RwLock}; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; @@ -16,15 +19,54 @@ use crate::index_controller::updates::*; #[allow(clippy::upper_case_acronyms)] type BEU64 = heed::zerocopy::U64; +struct IndexUuidUpdateIdCodec; + +#[repr(C)] +#[derive(Copy, Clone)] +struct IndexUuidUpdateId(Uuid, BEU64); + +// Is Uuid really zeroable (semantically)? +unsafe impl Zeroable for IndexUuidUpdateId {} +unsafe impl Pod for IndexUuidUpdateId {} + +impl IndexUuidUpdateId { + fn new(uuid: Uuid, update_id: u64) -> Self { + Self(uuid, BEU64::new(update_id)) + } +} + +const UUID_SIZE: usize = size_of::(); +const U64_SIZE: usize = size_of::(); + +impl<'a> BytesEncode<'a> for IndexUuidUpdateIdCodec { + type EItem = IndexUuidUpdateId; + + fn bytes_encode(item: &'a Self::EItem) -> Option> { + let bytes = bytemuck::cast_ref::(item); + Some(Cow::Borrowed(&bytes[..])) + } +} + +impl<'a> BytesDecode<'a> for IndexUuidUpdateIdCodec { + type DItem = (Uuid, u64); + + fn bytes_decode(bytes: &'a [u8]) -> Option { + let bytes = bytes.try_into().ok()?; + let IndexUuidUpdateId(uuid, id) = + bytemuck::cast_ref::<[u8; UUID_SIZE + U64_SIZE], IndexUuidUpdateId>(bytes); + Some((*uuid, id.get())) + } +} + #[derive(Clone)] pub struct UpdateStore { pub env: Env, - pending_meta: Database, SerdeJson>>, - pending: Database, SerdeJson>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, - processing: Arc>>>, + pending_meta: Database>>, + pending: Database>, + processed_meta: Database>>, + failed_meta: Database>>, + aborted_meta: Database>>, + processing: Arc)>>>, notification_sender: mpsc::Sender<()>, /// A lock on the update loop. This is meant to prevent a snapshot to occur while an update is /// processing, while not preventing writes all together during an update @@ -34,6 +76,7 @@ pub struct UpdateStore { pub trait HandleUpdate { fn handle_update( &mut self, + index_uuid: Uuid, meta: Processing, content: File, ) -> anyhow::Result, Failed>>; @@ -41,14 +84,15 @@ pub trait HandleUpdate { impl HandleUpdate for F where - F: FnMut(Processing, File) -> anyhow::Result, Failed>>, + F: FnMut(Uuid, Processing, File) -> anyhow::Result, Failed>>, { fn handle_update( &mut self, + index_uuid: Uuid, meta: Processing, content: File, ) -> anyhow::Result, Failed>> { - self(meta, content) + self(index_uuid, meta, content) } } @@ -131,24 +175,35 @@ where } /// Returns the new biggest id to use to store the new update. - fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { + fn new_update_id(&self, txn: &heed::RoTxn, index_uuid: Uuid) -> heed::Result { + // TODO: this is a very inneficient process for finding the next update id for each index, + // and needs to be made better. let last_pending = self .pending_meta .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); + .prefix_iter(txn, index_uuid.as_bytes())? + .remap_key_type::() + .last() + .transpose()? + .map(|((_, id), _)| id); let last_processed = self .processed_meta .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); + .prefix_iter(txn, index_uuid.as_bytes())? + .remap_key_type::() + .last() + .transpose()? + .map(|((_, id), _)| id); let last_aborted = self .aborted_meta .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); + .prefix_iter(txn, index_uuid.as_bytes())? + .remap_key_type::() + .last() + .transpose()? + .map(|((_, id), _)| id); let last_update_id = [last_pending, last_processed, last_aborted] .iter() @@ -176,13 +231,16 @@ where // no other update can have the same id because we use a write txn before // asking for the id and registering it so other update registering // will be forced to wait for a new write txn. - let update_id = self.new_update_id(&wtxn)?; - let update_key = BEU64::new(update_id); + let update_id = self.new_update_id(&wtxn, index_uuid)?; + let meta = Enqueued::new(meta, update_id); + let key = IndexUuidUpdateId::new(index_uuid, update_id); + self.pending_meta + .remap_key_type::() + .put(&mut wtxn, &key, &meta)?; - let meta = Enqueued::new(meta, update_id, index_uuid); - self.pending_meta.put(&mut wtxn, &update_key, &meta)?; self.pending - .put(&mut wtxn, &update_key, &content.as_ref().to_owned())?; + .remap_key_type::() + .put(&mut wtxn, &key, &content.as_ref().to_owned())?; wtxn.commit()?; @@ -191,6 +249,7 @@ where .expect("Update store loop exited."); Ok(meta) } + /// Executes the user provided function on the next pending update (the one with the lowest id). /// This is asynchronous as it let the user process the update with a read-only txn and /// only writing the result meta to the processed-meta store *after* it has been processed. @@ -201,25 +260,31 @@ where let _lock = self.update_lock.lock(); // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; - let first_meta = self.pending_meta.first(&rtxn)?; + + let first_meta = self + .pending_meta + .remap_key_type::() + .first(&rtxn)?; // If there is a pending update we process and only keep // a reader while processing it, not a writer. match first_meta { - Some((first_id, pending)) => { + Some(((index_uuid, update_id), pending)) => { + let key = IndexUuidUpdateId::new(index_uuid, update_id); let content_path = self .pending - .get(&rtxn, &first_id)? + .remap_key_type::() + .get(&rtxn, &key)? .expect("associated update content"); // we change the state of the update from pending to processing before we pass it // to the update handler. Processing store is non persistent to be able recover // from a failure let processing = pending.processing(); - self.processing.write().replace(processing.clone()); + self.processing.write().replace((index_uuid, processing.clone())); let file = File::open(&content_path)?; // Process the pending update using the provided user function. - let result = handler.handle_update(processing, file)?; + let result = handler.handle_update(index_uuid, processing, file)?; drop(rtxn); // Once the pending update have been successfully processed @@ -227,12 +292,24 @@ where // write the *new* meta to the processed-meta store and commit. let mut wtxn = self.env.write_txn()?; self.processing.write().take(); - self.pending_meta.delete(&mut wtxn, &first_id)?; + self.pending_meta + .remap_key_type::() + .delete(&mut wtxn, &key)?; + remove_file(&content_path)?; - self.pending.delete(&mut wtxn, &first_id)?; + + self.pending + .remap_key_type::() + .delete(&mut wtxn, &key)?; match result { - Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, - Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, + Ok(processed) => self + .processed_meta + .remap_key_type::() + .put(&mut wtxn, &key, &processed)?, + Err(failed) => self + .failed_meta + .remap_key_type::() + .put(&mut wtxn, &key, &failed)?, } wtxn.commit()?; @@ -242,28 +319,30 @@ where } } - pub fn list(&self) -> anyhow::Result>> { + pub fn list(&self, index_uuid: Uuid) -> anyhow::Result>> { let rtxn = self.env.read_txn()?; let mut updates = Vec::new(); let processing = self.processing.read(); - if let Some(ref processing) = *processing { - let update = UpdateStatus::from(processing.clone()); - updates.push(update); + if let Some((uuid, ref processing)) = *processing { + if uuid == index_uuid { + let update = UpdateStatus::from(processing.clone()); + updates.push(update); + } } let pending = self .pending_meta - .iter(&rtxn)? + .prefix_iter(&rtxn, index_uuid.as_bytes())? .filter_map(Result::ok) - .filter_map(|(_, p)| (Some(p.id()) != processing.as_ref().map(|p| p.id())).then(|| p)) + .filter_map(|(_, p)| (Some(p.id()) != processing.as_ref().map(|p| p.1.id())).then(|| p)) .map(UpdateStatus::from); updates.extend(pending); let aborted = self .aborted_meta - .iter(&rtxn)? + .prefix_iter(&rtxn, index_uuid.as_bytes())? .filter_map(Result::ok) .map(|(_, p)| p) .map(UpdateStatus::from); @@ -294,29 +373,49 @@ where } /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta(&self, update_id: u64) -> heed::Result>> { + pub fn meta( + &self, + index_uuid: Uuid, + update_id: u64, + ) -> heed::Result>> { let rtxn = self.env.read_txn()?; - let key = BEU64::new(update_id); + let key = IndexUuidUpdateId::new(index_uuid, update_id); - if let Some(ref meta) = *self.processing.read() { - if meta.id() == update_id { + if let Some((uuid, ref meta)) = *self.processing.read() { + if uuid == index_uuid && meta.id() == update_id { return Ok(Some(UpdateStatus::Processing(meta.clone()))); } } - if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { + if let Some(meta) = self + .pending_meta + .remap_key_type::() + .get(&rtxn, &key)? + { return Ok(Some(UpdateStatus::Enqueued(meta))); } - if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { + if let Some(meta) = self + .processed_meta + .remap_key_type::() + .get(&rtxn, &key)? + { return Ok(Some(UpdateStatus::Processed(meta))); } - if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { + if let Some(meta) = self + .aborted_meta + .remap_key_type::() + .get(&rtxn, &key)? + { return Ok(Some(UpdateStatus::Aborted(meta))); } - if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { + if let Some(meta) = self + .failed_meta + .remap_key_type::() + .get(&rtxn, &key)? + { return Ok(Some(UpdateStatus::Failed(meta))); } @@ -330,25 +429,45 @@ where /// that as already been processed or which doesn't actually exist, will /// return `None`. #[allow(dead_code)] - pub fn abort_update(&self, update_id: u64) -> heed::Result>> { + pub fn abort_update( + &self, + index_uuid: Uuid, + update_id: u64, + ) -> heed::Result>> { let mut wtxn = self.env.write_txn()?; - let key = BEU64::new(update_id); + let key = IndexUuidUpdateId::new(index_uuid, update_id); // We cannot abort an update that is currently being processed. - if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { + if self + .pending_meta + .remap_key_type::() + .first(&wtxn)? + .map(|((_, id), _)| id) + == Some(update_id) + { return Ok(None); } - let pending = match self.pending_meta.get(&wtxn, &key)? { + let pending = match self + .pending_meta + .remap_key_type::() + .get(&wtxn, &key)? + { Some(meta) => meta, None => return Ok(None), }; let aborted = pending.abort(); - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; + self.aborted_meta + .remap_key_type::() + .put(&mut wtxn, &key, &aborted)?; + self.pending_meta + .remap_key_type::() + .delete(&mut wtxn, &key)?; + self.pending + .remap_key_type::() + .delete(&mut wtxn, &key)?; wtxn.commit()?; @@ -358,22 +477,32 @@ where /// Aborts all the pending updates, and not the one being currently processed. /// Returns the update metas and ids that were successfully aborted. #[allow(dead_code)] - pub fn abort_pendings(&self) -> heed::Result)>> { + pub fn abort_pendings(&self, index_uuid: Uuid) -> heed::Result)>> { let mut wtxn = self.env.write_txn()?; let mut aborted_updates = Vec::new(); // We skip the first pending update as it is currently being processed. - for result in self.pending_meta.iter(&wtxn)?.skip(1) { - let (key, pending) = result?; - let id = key.get(); - aborted_updates.push((id, pending.abort())); + for result in self + .pending_meta + .prefix_iter(&wtxn, index_uuid.as_bytes())? + .remap_key_type::() + .skip(1) + { + let ((_, update_id), pending) = result?; + aborted_updates.push((update_id, pending.abort())); } for (id, aborted) in &aborted_updates { - let key = BEU64::new(*id); - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; + let key = IndexUuidUpdateId::new(index_uuid, *id); + self.aborted_meta + .remap_key_type::() + .put(&mut wtxn, &key, &aborted)?; + self.pending_meta + .remap_key_type::() + .delete(&mut wtxn, &key)?; + self.pending + .remap_key_type::() + .delete(&mut wtxn, &key)?; } wtxn.commit()?; diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 42712a396..60db5d3df 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -8,16 +8,14 @@ pub struct Enqueued { pub update_id: u64, pub meta: M, pub enqueued_at: DateTime, - pub index_uuid: Uuid, } impl Enqueued { - pub fn new(meta: M, update_id: u64, index_uuid: Uuid) -> Self { + pub fn new(meta: M, update_id: u64) -> Self { Self { enqueued_at: Utc::now(), meta, update_id, - index_uuid, } } From 9ce68d11a7d11120d36bf5082951d753e43cc183 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 13 Apr 2021 17:14:02 +0200 Subject: [PATCH 287/522] single update store instance --- .../src/index_controller/index_actor/actor.rs | 20 ++-- .../index_actor/handle_impl.rs | 3 +- .../index_controller/index_actor/message.rs | 1 + .../src/index_controller/index_actor/mod.rs | 1 + meilisearch-http/src/index_controller/mod.rs | 1 - .../index_controller/update_actor/actor.rs | 98 ++++++++-------- .../update_actor/handle_impl.rs | 12 +- .../index_controller/update_actor/message.rs | 4 - .../src/index_controller/update_actor/mod.rs | 5 - .../index_controller/update_actor/store.rs | 111 ------------------ .../update_actor/update_store.rs | 36 +++++- .../src/index_controller/updates.rs | 5 - 12 files changed, 95 insertions(+), 202 deletions(-) delete mode 100644 meilisearch-http/src/index_controller/update_actor/store.rs diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 9cca6557b..7278918aa 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -103,8 +103,8 @@ impl IndexActor { } => { let _ = ret.send(self.handle_create_index(uuid, primary_key).await); } - Update { ret, meta, data } => { - let _ = ret.send(self.handle_update(meta, data).await); + Update { ret, meta, data, uuid } => { + let _ = ret.send(self.handle_update(uuid, meta, data).await); } Search { ret, query, uuid } => { let _ = ret.send(self.handle_search(uuid, query).await); @@ -180,25 +180,25 @@ impl IndexActor { async fn handle_update( &self, + uuid: Uuid, meta: Processing, data: File, ) -> Result { - async fn get_result(actor: &IndexActor, meta: Processing, data: File) -> Result { + let get_result = || async { debug!("Processing update {}", meta.id()); - let uuid = *meta.index_uuid(); - let update_handler = actor.update_handler.clone(); - let index = match actor.store.get(uuid).await? { + let update_handler = self.update_handler.clone(); + let index = match self.store.get(uuid).await? { Some(index) => index, - None => actor.store.create(uuid, None).await?, + None => self.store.create(uuid, None).await?, }; spawn_blocking(move || update_handler.handle_update(meta, data, index)) .await .map_err(|e| IndexError::Error(e.into())) - } + }; - *self.processing.write().await = Some(*meta.index_uuid()); - let result = get_result(self, meta, data).await; + *self.processing.write().await = Some(uuid); + let result = get_result().await; *self.processing.write().await = None; result diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 93406c13b..5a734cdbf 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -32,11 +32,12 @@ impl IndexActorHandle for IndexActorHandleImpl { async fn update( &self, + uuid: Uuid, meta: Processing, data: std::fs::File, ) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Update { ret, meta, data }; + let msg = IndexMsg::Update { ret, meta, data, uuid }; let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 6da0f8628..cb28f2868 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -15,6 +15,7 @@ pub enum IndexMsg { ret: oneshot::Sender>, }, Update { + uuid: Uuid, meta: Processing, data: std::fs::File, ret: oneshot::Sender>, diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 426eb29e4..70a69822d 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -75,6 +75,7 @@ pub trait IndexActorHandle { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn update( &self, + uuid: Uuid, meta: Processing, data: std::fs::File, ) -> anyhow::Result; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 8361c45cc..cf8d036c7 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -233,7 +233,6 @@ impl IndexController { let uid = uid.ok_or_else(|| anyhow::anyhow!("Can't create an index without a uid."))?; let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; - let _ = self.update_handle.create(uuid).await?; let meta = IndexMetadata { name: uid.clone(), uid, diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index f725dda84..8cf86d126 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -1,5 +1,6 @@ use std::io::SeekFrom; use std::path::{Path, PathBuf}; +use std::sync::Arc; use log::info; use oxidized_json_checker::JsonChecker; @@ -8,31 +9,44 @@ use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio::sync::mpsc; use uuid::Uuid; +use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore}; use crate::index_controller::index_actor::IndexActorHandle; -use crate::index_controller::{get_arc_ownership_blocking, UpdateMeta, UpdateStatus}; +use crate::index_controller::{UpdateMeta, UpdateStatus}; -use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStoreStore}; -pub struct UpdateActor { +pub struct UpdateActor { path: PathBuf, - store: S, + store: Arc, inbox: mpsc::Receiver>, index_handle: I, } -impl UpdateActor +impl UpdateActor where D: AsRef<[u8]> + Sized + 'static, - S: UpdateStoreStore, I: IndexActorHandle + Clone + Send + Sync + 'static, { pub fn new( - store: S, + update_db_size: usize, inbox: mpsc::Receiver>, path: impl AsRef, index_handle: I, ) -> anyhow::Result { - let path = path.as_ref().to_owned(); + let path = path + .as_ref() + .to_owned() + .join("updates"); + + std::fs::create_dir_all(&path)?; + + let mut options = heed::EnvOpenOptions::new(); + options.map_size(update_db_size); + + let handle = index_handle.clone(); + let store = UpdateStore::open(options, &path, move |uuid, meta, file| { + futures::executor::block_on(handle.update(uuid, meta, file)) + }) + .map_err(|e| UpdateError::Error(e.into()))?; std::fs::create_dir_all(path.join("update_files"))?; assert!(path.exists()); Ok(Self { @@ -67,9 +81,6 @@ where Some(Delete { uuid, ret }) => { let _ = ret.send(self.handle_delete(uuid).await); } - Some(Create { uuid, ret }) => { - let _ = ret.send(self.handle_create(uuid).await); - } Some(Snapshot { uuid, path, ret }) => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } @@ -87,7 +98,6 @@ where meta: UpdateMeta, mut payload: mpsc::Receiver>, ) -> Result { - let update_store = self.store.get_or_create(uuid).await?; let update_file_id = uuid::Uuid::new_v4(); let path = self .path @@ -123,6 +133,8 @@ where let mut file = file.into_std().await; + let update_store = self.store.clone(); + tokio::task::spawn_blocking(move || { use std::io::{copy, sink, BufReader, Seek}; @@ -157,11 +169,10 @@ where } async fn handle_list_updates(&self, uuid: Uuid) -> Result> { - let update_store = self.store.get(uuid).await?; + let update_store = self.store.clone(); tokio::task::spawn_blocking(move || { let result = update_store - .ok_or(UpdateError::UnexistingIndex(uuid))? - .list() + .list(uuid) .map_err(|e| UpdateError::Error(e.into()))?; Ok(result) }) @@ -170,60 +181,45 @@ where } async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { - let store = self - .store - .get(uuid) - .await? - .ok_or(UpdateError::UnexistingIndex(uuid))?; + let store = self.store.clone(); let result = store - .meta(id) + .meta(uuid, id) .map_err(|e| UpdateError::Error(Box::new(e)))? .ok_or(UpdateError::UnexistingUpdate(id))?; Ok(result) } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let store = self.store.delete(uuid).await?; + let store = self.store.clone(); - if let Some(store) = store { - tokio::task::spawn(async move { - let store = get_arc_ownership_blocking(store).await; - tokio::task::spawn_blocking(move || { - store.prepare_for_closing().wait(); - info!("Update store {} was closed.", uuid); - }); - }); - } + tokio::task::spawn_blocking(move || store.delete_all(uuid)) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?; Ok(()) } - async fn handle_create(&self, uuid: Uuid) -> Result<()> { - let _ = self.store.get_or_create(uuid).await?; - Ok(()) - } - async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let index_handle = self.index_handle.clone(); - if let Some(update_store) = self.store.get(uuid).await? { - tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - // acquire write lock to prevent further writes during snapshot - // the update lock must be acquired BEFORE the write lock to prevent dead lock - let _lock = update_store.update_lock.lock(); - let mut txn = update_store.env.write_txn()?; + let update_store = self.store.clone(); + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + // acquire write lock to prevent further writes during snapshot + // the update lock must be acquired BEFORE the write lock to prevent dead lock + let _lock = update_store.update_lock.lock(); + let mut txn = update_store.env.write_txn()?; - // create db snapshot - update_store.snapshot(&mut txn, &path, uuid)?; + // create db snapshot + update_store.snapshot(&mut txn, &path, uuid)?; - futures::executor::block_on( - async move { index_handle.snapshot(uuid, path).await }, - )?; - Ok(()) - }) - .await + futures::executor::block_on( + async move { index_handle.snapshot(uuid, path).await }, + )?; + Ok(()) + }) + .await .map_err(|e| UpdateError::Error(e.into()))? .map_err(|e| UpdateError::Error(e.into()))?; - } Ok(()) } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 860cc2bc8..bbb52d17c 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::index_controller::IndexActorHandle; use super::{ - MapUpdateStoreStore, PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, + PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStatus, }; @@ -29,8 +29,7 @@ where { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); - let store = MapUpdateStoreStore::new(index_handle.clone(), &path, update_store_size); - let actor = UpdateActor::new(store, receiver, path, index_handle)?; + let actor = UpdateActor::new(update_store_size, receiver, path, index_handle)?; tokio::task::spawn(actor.run()); @@ -65,13 +64,6 @@ where receiver.await.expect("update actor killed.") } - async fn create(&self, uuid: Uuid) -> Result<()> { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Create { uuid, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Snapshot { uuid, path, ret }; diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index f8150c00a..c863c803e 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -25,10 +25,6 @@ pub enum UpdateMsg { uuid: Uuid, ret: oneshot::Sender>, }, - Create { - uuid: Uuid, - ret: oneshot::Sender>, - }, Snapshot { uuid: Uuid, path: PathBuf, diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 228b47b02..1c9461f6a 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -1,7 +1,6 @@ mod actor; mod handle_impl; mod message; -mod store; mod update_store; use std::path::PathBuf; @@ -15,7 +14,6 @@ use crate::index_controller::{UpdateMeta, UpdateStatus}; use actor::UpdateActor; use message::UpdateMsg; -use store::{MapUpdateStoreStore, UpdateStoreStore}; pub use handle_impl::UpdateActorHandleImpl; @@ -30,8 +28,6 @@ use mockall::automock; pub enum UpdateError { #[error("error with update: {0}")] Error(Box), - #[error("Index {0} doesn't exist.")] - UnexistingIndex(Uuid), #[error("Update {0} doesn't exist.")] UnexistingUpdate(u64), } @@ -44,7 +40,6 @@ pub trait UpdateActorHandle { async fn get_all_updates_status(&self, uuid: Uuid) -> Result>; async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; - async fn create(&self, uuid: Uuid) -> Result<()>; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; async fn get_size(&self, uuid: Uuid) -> Result; async fn update( diff --git a/meilisearch-http/src/index_controller/update_actor/store.rs b/meilisearch-http/src/index_controller/update_actor/store.rs deleted file mode 100644 index 676182a62..000000000 --- a/meilisearch-http/src/index_controller/update_actor/store.rs +++ /dev/null @@ -1,111 +0,0 @@ -use std::collections::hash_map::Entry; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use tokio::fs; -use tokio::sync::RwLock; -use uuid::Uuid; - -use super::{Result, UpdateError, UpdateStore}; -use crate::index_controller::IndexActorHandle; - -#[async_trait::async_trait] -pub trait UpdateStoreStore { - async fn get_or_create(&self, uuid: Uuid) -> Result>; - async fn delete(&self, uuid: Uuid) -> Result>>; - async fn get(&self, uuid: Uuid) -> Result>>; -} - -pub struct MapUpdateStoreStore { - db: Arc>>>, - index_handle: I, - path: PathBuf, - update_store_size: usize, -} - -impl MapUpdateStoreStore { - pub fn new(index_handle: I, path: impl AsRef, update_store_size: usize) -> Self { - let db = Arc::new(RwLock::new(HashMap::new())); - let path = path.as_ref().to_owned(); - Self { - db, - index_handle, - path, - update_store_size, - } - } -} - -#[async_trait::async_trait] -impl UpdateStoreStore for MapUpdateStoreStore -where - I: IndexActorHandle + Clone + Send + Sync + 'static, -{ - async fn get_or_create(&self, uuid: Uuid) -> Result> { - match self.db.write().await.entry(uuid) { - Entry::Vacant(e) => { - let mut options = heed::EnvOpenOptions::new(); - let update_store_size = self.update_store_size; - options.map_size(update_store_size); - let path = self.path.clone().join(format!("updates-{}", e.key())); - fs::create_dir_all(&path).await.unwrap(); - let index_handle = self.index_handle.clone(); - let store = UpdateStore::open(options, &path, move |meta, file| { - futures::executor::block_on(index_handle.update(meta, file)) - }) - .map_err(|e| UpdateError::Error(e.into()))?; - let store = e.insert(store); - Ok(store.clone()) - } - Entry::Occupied(e) => Ok(e.get().clone()), - } - } - - async fn get(&self, uuid: Uuid) -> Result>> { - let guard = self.db.read().await; - match guard.get(&uuid) { - Some(uuid) => Ok(Some(uuid.clone())), - None => { - // The index is not found in the found in the loaded indexes, so we attempt to load - // it from disk. We need to acquire a write lock **before** attempting to open the - // index, because someone could be trying to open it at the same time as us. - drop(guard); - let path = self.path.clone().join(format!("updates-{}", uuid)); - if path.exists() { - let mut guard = self.db.write().await; - match guard.entry(uuid) { - Entry::Vacant(entry) => { - // We can safely load the index - let index_handle = self.index_handle.clone(); - let mut options = heed::EnvOpenOptions::new(); - let update_store_size = self.update_store_size; - options.map_size(update_store_size); - let store = UpdateStore::open(options, &path, move |meta, file| { - futures::executor::block_on(index_handle.update(meta, file)) - }) - .map_err(|e| UpdateError::Error(e.into()))?; - let store = entry.insert(store); - Ok(Some(store.clone())) - } - Entry::Occupied(entry) => { - // The index was loaded while we attempted to to iter - Ok(Some(entry.get().clone())) - } - } - } else { - Ok(None) - } - } - } - } - - async fn delete(&self, uuid: Uuid) -> Result>> { - let store = self.db.write().await.remove(&uuid); - let path = self.path.clone().join(format!("updates-{}", uuid)); - if store.is_some() || path.exists() { - fs::remove_dir_all(path).await.unwrap(); - } - Ok(store) - } -} diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index efd827619..c0c76283c 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -170,10 +170,6 @@ where Ok(update_store) } - pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { - self.env.prepare_for_closing() - } - /// Returns the new biggest id to use to store the new update. fn new_update_id(&self, txn: &heed::RoTxn, index_uuid: Uuid) -> heed::Result { // TODO: this is a very inneficient process for finding the next update id for each index, @@ -510,6 +506,38 @@ where Ok(aborted_updates) } + pub fn delete_all(&self, uuid: Uuid) -> anyhow::Result<()> { + fn delete_all( + txn: &mut heed::RwTxn, + uuid: Uuid, + db: Database + ) -> anyhow::Result<()> + where A: for<'a> heed::BytesDecode<'a> + { + let mut iter = db.prefix_iter_mut(txn, uuid.as_bytes())?; + while let Some(_) = iter.next() { + iter.del_current()?; + } + Ok(()) + } + + let mut txn = self.env.write_txn()?; + + delete_all(&mut txn, uuid, self.pending)?; + delete_all(&mut txn, uuid, self.pending_meta)?; + delete_all(&mut txn, uuid, self.processed_meta)?; + delete_all(&mut txn, uuid, self.aborted_meta)?; + delete_all(&mut txn, uuid, self.failed_meta)?; + + let processing = self.processing.upgradable_read(); + if let Some((processing_uuid, _)) = *processing { + if processing_uuid == uuid { + parking_lot::RwLockUpgradableReadGuard::upgrade(processing).take(); + } + } + Ok(()) + } + pub fn snapshot( &self, txn: &mut heed::RwTxn, diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 60db5d3df..36e327cc2 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -1,6 +1,5 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -74,10 +73,6 @@ impl Processing { self.from.meta() } - pub fn index_uuid(&self) -> &Uuid { - &self.from.index_uuid - } - pub fn process(self, meta: N) -> Processed { Processed { success: meta, From b626d02ffe0a6e1f0b1cc0e3fc2e9daff6f00605 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 13 Apr 2021 17:46:39 +0200 Subject: [PATCH 288/522] simplify index actor run loop --- .../src/index_controller/index_actor/actor.rs | 48 ++++--------------- .../index_actor/handle_impl.rs | 31 ++++++------ 2 files changed, 24 insertions(+), 55 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 7278918aa..af755f65f 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -1,10 +1,8 @@ use std::fs::File; -use std::future::Future; use std::path::PathBuf; use std::sync::Arc; use async_stream::stream; -use futures::pin_mut; use futures::stream::StreamExt; use heed::CompactionOption; use log::debug; @@ -22,8 +20,7 @@ use crate::option::IndexerOpts; use super::{IndexError, IndexMeta, IndexMsg, IndexSettings, IndexStore, Result, UpdateResult}; pub struct IndexActor { - read_receiver: Option>, - write_receiver: Option>, + receiver: Option>, update_handler: Arc, processing: RwLock>, store: S, @@ -31,18 +28,16 @@ pub struct IndexActor { impl IndexActor { pub fn new( - read_receiver: mpsc::Receiver, - write_receiver: mpsc::Receiver, + receiver: mpsc::Receiver, store: S, ) -> Result { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; let update_handler = Arc::new(update_handler); - let read_receiver = Some(read_receiver); - let write_receiver = Some(write_receiver); + let receiver = Some(receiver); Ok(Self { - read_receiver, - write_receiver, + receiver, + store, update_handler, processing: RwLock::new(None), store, @@ -53,44 +48,21 @@ impl IndexActor { /// through the read channel are processed concurrently, the messages sent through the write /// channel are processed one at a time. pub async fn run(mut self) { - let mut read_receiver = self - .read_receiver + let mut receiver = self + .receiver .take() .expect("Index Actor must have a inbox at this point."); - let read_stream = stream! { + let stream = stream! { loop { - match read_receiver.recv().await { + match receiver.recv().await { Some(msg) => yield msg, None => break, } } }; - let mut write_receiver = self - .write_receiver - .take() - .expect("Index Actor must have a inbox at this point."); - - let write_stream = stream! { - loop { - match write_receiver.recv().await { - Some(msg) => yield msg, - None => break, - } - } - }; - - pin_mut!(write_stream); - pin_mut!(read_stream); - - let fut1 = read_stream.for_each_concurrent(Some(10), |msg| self.handle_message(msg)); - let fut2 = write_stream.for_each_concurrent(Some(1), |msg| self.handle_message(msg)); - - let fut1: Box + Unpin + Send> = Box::new(fut1); - let fut2: Box + Unpin + Send> = Box::new(fut2); - - tokio::join!(fut1, fut2); + stream.for_each_concurrent(Some(10), |msg| self.handle_message(msg)).await; } async fn handle_message(&self, msg: IndexMsg) { diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 5a734cdbf..629fd0db7 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -13,8 +13,7 @@ use super::{ #[derive(Clone)] pub struct IndexActorHandleImpl { - read_sender: mpsc::Sender, - write_sender: mpsc::Sender, + sender: mpsc::Sender, } #[async_trait::async_trait] @@ -26,7 +25,7 @@ impl IndexActorHandle for IndexActorHandleImpl { uuid, primary_key, }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } @@ -38,21 +37,21 @@ impl IndexActorHandle for IndexActorHandleImpl { ) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Update { ret, meta, data, uuid }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Search { uuid, query, ret }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } async fn settings(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Settings { uuid, ret }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -71,7 +70,7 @@ impl IndexActorHandle for IndexActorHandleImpl { attributes_to_retrieve, limit, }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -88,21 +87,21 @@ impl IndexActorHandle for IndexActorHandleImpl { doc_id, attributes_to_retrieve, }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } async fn delete(&self, uuid: Uuid) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Delete { uuid, ret }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } async fn get_index_meta(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetMeta { uuid, ret }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -113,14 +112,14 @@ impl IndexActorHandle for IndexActorHandleImpl { index_settings, ret, }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Snapshot { uuid, path, ret }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -134,15 +133,13 @@ impl IndexActorHandle for IndexActorHandleImpl { impl IndexActorHandleImpl { pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { - let (read_sender, read_receiver) = mpsc::channel(100); - let (write_sender, write_receiver) = mpsc::channel(100); + let (sender, receiver) = mpsc::channel(100); let store = MapIndexStore::new(path, index_size); - let actor = IndexActor::new(read_receiver, write_receiver, store)?; + let actor = IndexActor::new(receiver, store)?; tokio::task::spawn(actor.run()); Ok(Self { - read_sender, - write_sender, + sender, }) } } From 2b154524bbeef1a51dc9726545c0e59842803672 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 13 Apr 2021 18:11:37 +0200 Subject: [PATCH 289/522] fix filtered out pending update --- .../index_controller/update_actor/update_store.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index c0c76283c..a8f5fe800 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -331,7 +331,18 @@ where .pending_meta .prefix_iter(&rtxn, index_uuid.as_bytes())? .filter_map(Result::ok) - .filter_map(|(_, p)| (Some(p.id()) != processing.as_ref().map(|p| p.1.id())).then(|| p)) + .filter_map(|(_, p)| { + if let Some((uuid, ref processing)) = *processing { + // Filter out the currently processing update if it is from this index. + if uuid == index_uuid && processing.id() == p.id() { + None + } else { + Some(p) + } + } else { + Some(p) + } + }) .map(UpdateStatus::from); updates.extend(pending); From 33830d5ecf1bc8defec4983d466530ed14bcd046 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 14 Apr 2021 17:53:12 +0200 Subject: [PATCH 290/522] fix snapshots --- meilisearch-http/src/data/mod.rs | 2 +- meilisearch-http/src/index/mod.rs | 2 +- .../src/index_controller/index_actor/actor.rs | 19 ++++-- .../index_actor/handle_impl.rs | 13 ++-- .../src/index_controller/index_actor/mod.rs | 1 + meilisearch-http/src/index_controller/mod.rs | 6 +- .../src/index_controller/snapshot.rs | 11 +--- .../index_controller/update_actor/actor.rs | 64 ++++++++++--------- .../update_actor/handle_impl.rs | 13 ++-- .../index_controller/update_actor/message.rs | 3 +- .../src/index_controller/update_actor/mod.rs | 4 +- .../update_actor/update_store.rs | 28 +++++--- meilisearch-http/src/routes/document.rs | 24 +++---- meilisearch-http/src/routes/settings/mod.rs | 12 ++-- .../tests/settings/get_settings.rs | 8 +-- 15 files changed, 109 insertions(+), 101 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 3aee18217..ab50e83cc 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -127,7 +127,7 @@ impl Data { stats.database_size += index_stats.size; stats.database_size += self .index_controller - .get_updates_size(index.uid.clone()) + .get_updates_size() .await?; stats.last_update = Some(match stats.last_update { diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 57a939aaa..0bb20d3ed 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -6,9 +6,9 @@ use anyhow::{bail, Context}; use milli::obkv_to_json; use serde_json::{Map, Value}; +use crate::helpers::EnvSizer; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use updates::{Facets, Settings, UpdateResult}; -use crate::helpers::EnvSizer; mod search; mod updates; diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index af755f65f..06aa90562 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -19,6 +19,8 @@ use crate::option::IndexerOpts; use super::{IndexError, IndexMeta, IndexMsg, IndexSettings, IndexStore, Result, UpdateResult}; +pub const CONCURRENT_INDEX_MSG: usize = 10; + pub struct IndexActor { receiver: Option>, update_handler: Arc, @@ -27,10 +29,7 @@ pub struct IndexActor { } impl IndexActor { - pub fn new( - receiver: mpsc::Receiver, - store: S, - ) -> Result { + pub fn new(receiver: mpsc::Receiver, store: S) -> Result { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; let update_handler = Arc::new(update_handler); @@ -40,7 +39,6 @@ impl IndexActor { store, update_handler, processing: RwLock::new(None), - store, }) } @@ -62,7 +60,9 @@ impl IndexActor { } }; - stream.for_each_concurrent(Some(10), |msg| self.handle_message(msg)).await; + stream + .for_each_concurrent(Some(CONCURRENT_INDEX_MSG), |msg| self.handle_message(msg)) + .await; } async fn handle_message(&self, msg: IndexMsg) { @@ -75,7 +75,12 @@ impl IndexActor { } => { let _ = ret.send(self.handle_create_index(uuid, primary_key).await); } - Update { ret, meta, data, uuid } => { + Update { + ret, + meta, + data, + uuid, + } => { let _ = ret.send(self.handle_update(uuid, meta, data).await); } Search { ret, query, uuid } => { diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 629fd0db7..d23357d38 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -36,7 +36,12 @@ impl IndexActorHandle for IndexActorHandleImpl { data: std::fs::File, ) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Update { ret, meta, data, uuid }; + let msg = IndexMsg::Update { + ret, + meta, + data, + uuid, + }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -126,7 +131,7 @@ impl IndexActorHandle for IndexActorHandleImpl { async fn get_index_stats(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetStats { uuid, ret }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } } @@ -138,8 +143,6 @@ impl IndexActorHandleImpl { let store = MapIndexStore::new(path, index_size); let actor = IndexActor::new(receiver, store)?; tokio::task::spawn(actor.run()); - Ok(Self { - sender, - }) + Ok(Self { sender }) } } diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 70a69822d..e44891b5a 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -8,6 +8,7 @@ use thiserror::Error; use uuid::Uuid; use actor::IndexActor; +pub use actor::CONCURRENT_INDEX_MSG; pub use handle_impl::IndexActorHandleImpl; use message::IndexMsg; use store::{IndexStore, MapIndexStore}; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index cf8d036c7..9d9ed02c1 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -356,10 +356,8 @@ impl IndexController { Ok(self.index_handle.get_index_stats(uuid).await?) } - pub async fn get_updates_size(&self, uid: String) -> anyhow::Result { - let uuid = self.uuid_resolver.get(uid.clone()).await?; - - Ok(self.update_handle.get_size(uuid).await?) + pub async fn get_updates_size(&self) -> anyhow::Result { + Ok(self.update_handle.get_size().await?) } pub async fn get_uuids_size(&self) -> anyhow::Result { diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 8557fe04e..eba5a5f3b 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -71,15 +71,10 @@ where return Ok(()); } - let tasks = uuids - .iter() - .map(|&uuid| { - self.update_handle - .snapshot(uuid, temp_snapshot_path.clone()) - }) - .collect::>(); - futures::future::try_join_all(tasks).await?; + self.update_handle + .snapshot(uuids, temp_snapshot_path.clone()) + .await?; let snapshot_dir = self.snapshot_path.clone(); let snapshot_path = self diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 8cf86d126..2a752d0b3 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -8,12 +8,12 @@ use tokio::fs; use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio::sync::mpsc; use uuid::Uuid; +use futures::StreamExt; use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore}; -use crate::index_controller::index_actor::IndexActorHandle; +use crate::index_controller::index_actor::{IndexActorHandle, CONCURRENT_INDEX_MSG}; use crate::index_controller::{UpdateMeta, UpdateStatus}; - pub struct UpdateActor { path: PathBuf, store: Arc, @@ -32,10 +32,7 @@ where path: impl AsRef, index_handle: I, ) -> anyhow::Result { - let path = path - .as_ref() - .to_owned() - .join("updates"); + let path = path.as_ref().to_owned().join("updates"); std::fs::create_dir_all(&path)?; @@ -81,11 +78,11 @@ where Some(Delete { uuid, ret }) => { let _ = ret.send(self.handle_delete(uuid).await); } - Some(Snapshot { uuid, path, ret }) => { - let _ = ret.send(self.handle_snapshot(uuid, path).await); + Some(Snapshot { uuids, path, ret }) => { + let _ = ret.send(self.handle_snapshot(uuids, path).await); } - Some(GetSize { uuid, ret }) => { - let _ = ret.send(self.handle_get_size(uuid).await); + Some(GetSize { ret }) => { + let _ = ret.send(self.handle_get_size().await); } None => break, } @@ -200,7 +197,7 @@ where Ok(()) } - async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + async fn handle_snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { @@ -210,32 +207,41 @@ where let mut txn = update_store.env.write_txn()?; // create db snapshot - update_store.snapshot(&mut txn, &path, uuid)?; + update_store.snapshot(&mut txn, &path)?; - futures::executor::block_on( - async move { index_handle.snapshot(uuid, path).await }, - )?; - Ok(()) + // Perform the snapshot of each index concurently. Only a third of the capabilities of + // the index actor at a time not to put too much pressure on the index actor + let path = &path; + let handle = &index_handle; + + let mut stream = futures::stream::iter(uuids.iter()) + .map(|&uuid| handle.snapshot(uuid, path.clone())) + .buffer_unordered(CONCURRENT_INDEX_MSG / 3); + + futures::executor::block_on(async { + while let Some(res) = stream.next().await { + res?; + } + Ok(()) + }) }) .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?; + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?; Ok(()) } - async fn handle_get_size(&self, uuid: Uuid) -> Result { - let size = match self.store.get(uuid).await? { - Some(update_store) => tokio::task::spawn_blocking(move || -> anyhow::Result { - let txn = update_store.env.read_txn()?; + async fn handle_get_size(&self) -> Result { + let update_store = self.store.clone(); + let size = tokio::task::spawn_blocking(move || -> anyhow::Result { + let txn = update_store.env.read_txn()?; - update_store.get_size(&txn) - }) - .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?, - None => 0, - }; + update_store.get_size(&txn) + }) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?; Ok(size) } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index bbb52d17c..5c5a3e051 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -6,8 +6,7 @@ use uuid::Uuid; use crate::index_controller::IndexActorHandle; use super::{ - PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, - UpdateMsg, UpdateStatus, + PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStatus, }; #[derive(Clone)] @@ -27,7 +26,7 @@ where where I: IndexActorHandle + Clone + Send + Sync + 'static, { - let path = path.as_ref().to_owned().join("updates"); + let path = path.as_ref().to_owned(); let (sender, receiver) = mpsc::channel(100); let actor = UpdateActor::new(update_store_size, receiver, path, index_handle)?; @@ -64,16 +63,16 @@ where receiver.await.expect("update actor killed.") } - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Snapshot { uuid, path, ret }; + let msg = UpdateMsg::Snapshot { uuids, path, ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } - async fn get_size(&self, uuid: Uuid) -> Result { + async fn get_size(&self) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::GetSize { uuid, ret }; + let msg = UpdateMsg::GetSize { ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index c863c803e..0f0005862 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -26,12 +26,11 @@ pub enum UpdateMsg { ret: oneshot::Sender>, }, Snapshot { - uuid: Uuid, + uuids: Vec, path: PathBuf, ret: oneshot::Sender>, }, GetSize { - uuid: Uuid, ret: oneshot::Sender>, }, } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 1c9461f6a..0f815dbf3 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -40,8 +40,8 @@ pub trait UpdateActorHandle { async fn get_all_updates_status(&self, uuid: Uuid) -> Result>; async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; - async fn get_size(&self, uuid: Uuid) -> Result; + async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()>; + async fn get_size(&self) -> Result; async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index a8f5fe800..79693ba1f 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -5,6 +5,7 @@ use std::mem::size_of; use std::path::{Path, PathBuf}; use std::sync::Arc; +use anyhow::Context; use bytemuck::{Pod, Zeroable}; use heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; use heed::{BytesDecode, BytesEncode, CompactionOption, Database, Env, EnvOpenOptions}; @@ -106,7 +107,7 @@ where mut options: EnvOpenOptions, path: P, update_handler: U, - ) -> heed::Result> + ) -> anyhow::Result> where P: AsRef, U: HandleUpdate + Sync + Clone + Send + 'static, @@ -127,6 +128,11 @@ where let update_lock = Arc::new(Mutex::new(())); + // Init update loop to perform any pending updates at launch. + // Since we just launched the update store, and we still own the receiving end of the + // channel, this call is guarenteed to succeed. + notification_sender.try_send(()).expect("Failed to init update store"); + let update_store = Arc::new(UpdateStore { env, pending, @@ -277,8 +283,11 @@ where // to the update handler. Processing store is non persistent to be able recover // from a failure let processing = pending.processing(); - self.processing.write().replace((index_uuid, processing.clone())); - let file = File::open(&content_path)?; + self.processing + .write() + .replace((index_uuid, processing.clone())); + let file = File::open(&content_path) + .with_context(|| format!("file at path: {:?}", &content_path))?; // Process the pending update using the provided user function. let result = handler.handle_update(index_uuid, processing, file)?; drop(rtxn); @@ -521,9 +530,10 @@ where fn delete_all( txn: &mut heed::RwTxn, uuid: Uuid, - db: Database + db: Database, ) -> anyhow::Result<()> - where A: for<'a> heed::BytesDecode<'a> + where + A: for<'a> heed::BytesDecode<'a>, { let mut iter = db.prefix_iter_mut(txn, uuid.as_bytes())?; while let Some(_) = iter.next() { @@ -553,19 +563,17 @@ where &self, txn: &mut heed::RwTxn, path: impl AsRef, - uuid: Uuid, ) -> anyhow::Result<()> { let update_path = path.as_ref().join("updates"); create_dir_all(&update_path)?; - let mut snapshot_path = update_path.join(format!("update-{}", uuid)); // acquire write lock to prevent further writes during snapshot - create_dir_all(&snapshot_path)?; - snapshot_path.push("data.mdb"); + create_dir_all(&update_path)?; + let db_path = update_path.join("data.mdb"); // create db snapshot self.env - .copy_to_path(&snapshot_path, CompactionOption::Enabled)?; + .copy_to_path(&db_path, CompactionOption::Enabled)?; let update_files_path = update_path.join("update_files"); create_dir_all(&update_files_path)?; diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index ed5d88230..357a2b16a 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -84,9 +84,9 @@ async fn delete_document( .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]) .await { - Ok(update_status) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) - } + Ok(update_status) => Ok( + HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })) + ), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -163,9 +163,9 @@ async fn add_documents( .await; match addition_result { - Ok(update_status) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) - } + Ok(update_status) => Ok( + HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })) + ), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -242,9 +242,9 @@ async fn delete_documents( .collect(); match data.delete_documents(path.index_uid.clone(), ids).await { - Ok(update_status) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) - } + Ok(update_status) => Ok( + HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })) + ), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -258,9 +258,9 @@ async fn clear_all_documents( path: web::Path, ) -> Result { match data.clear_documents(path.index_uid.clone()).await { - Ok(update_status) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) - } + Ok(update_status) => Ok( + HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })) + ), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 54cc53d7e..8ca51dbf5 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -143,9 +143,9 @@ async fn update_all( .update_settings(index_uid.into_inner(), body.into_inner(), true) .await { - Ok(update_result) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_result.id() }))) - } + Ok(update_result) => Ok( + HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_result.id() })) + ), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -175,9 +175,9 @@ async fn delete_all( .update_settings(index_uid.into_inner(), settings, false) .await { - Ok(update_result) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_result.id() }))) - } + Ok(update_result) => Ok( + HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_result.id() })) + ), Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 3412f45af..4230e19f8 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -23,13 +23,7 @@ async fn get_settings() { assert_eq!(settings["distinctAttribute"], json!(null)); assert_eq!( settings["rankingRules"], - json!([ - "words", - "typo", - "proximity", - "attribute", - "exactness" - ]) + json!(["words", "typo", "proximity", "attribute", "exactness"]) ); assert_eq!(settings["stopWords"], json!([])); } From ee675eadf13c65dc786fc1c1754d19f54917db1f Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 14 Apr 2021 18:55:04 +0200 Subject: [PATCH 291/522] fix stats --- meilisearch-http/src/data/mod.rs | 38 ++--------- .../src/index_controller/index_actor/actor.rs | 11 +--- meilisearch-http/src/index_controller/mod.rs | 66 +++++++++++++++---- .../index_controller/update_actor/actor.rs | 26 +++++--- .../update_actor/handle_impl.rs | 6 +- .../index_controller/update_actor/message.rs | 6 +- .../src/index_controller/update_actor/mod.rs | 10 ++- .../update_actor/update_store.rs | 2 +- meilisearch-http/src/routes/stats.rs | 49 +------------- 9 files changed, 97 insertions(+), 117 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index ab50e83cc..60fb5ae20 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -1,12 +1,10 @@ -use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; -use chrono::{DateTime, Utc}; use sha2::Digest; use crate::index::Settings; -use crate::index_controller::{IndexController, IndexStats}; +use crate::index_controller::{IndexController, IndexStats, Stats}; use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::option::Opt; @@ -39,13 +37,6 @@ pub struct ApiKeys { pub master: Option, } -#[derive(Default)] -pub struct Stats { - pub database_size: u64, - pub last_update: Option>, - pub indexes: HashMap, -} - impl ApiKeys { pub fn generate_missing_api_keys(&mut self) { if let Some(master_key) = &self.master { @@ -114,33 +105,14 @@ impl Data { } pub async fn get_index_stats(&self, uid: String) -> anyhow::Result { - Ok(self.index_controller.get_stats(uid).await?) + Ok(self.index_controller.get_index_stats(uid).await?) } - pub async fn get_stats(&self) -> anyhow::Result { - let mut stats = Stats::default(); - stats.database_size += self.index_controller.get_uuids_size().await?; - - for index in self.index_controller.list_indexes().await? { - let index_stats = self.index_controller.get_stats(index.uid.clone()).await?; - - stats.database_size += index_stats.size; - stats.database_size += self - .index_controller - .get_updates_size() - .await?; - - stats.last_update = Some(match stats.last_update { - Some(last_update) => last_update.max(index.meta.updated_at), - None => index.meta.updated_at, - }); - - stats.indexes.insert(index.uid, index_stats); - } - - Ok(stats) + pub async fn get_all_stats(&self) -> anyhow::Result { + Ok(self.index_controller.get_all_stats().await?) } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 06aa90562..ff7064600 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -6,7 +6,7 @@ use async_stream::stream; use futures::stream::StreamExt; use heed::CompactionOption; use log::debug; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::mpsc; use tokio::task::spawn_blocking; use uuid::Uuid; @@ -24,7 +24,6 @@ pub const CONCURRENT_INDEX_MSG: usize = 10; pub struct IndexActor { receiver: Option>, update_handler: Arc, - processing: RwLock>, store: S, } @@ -38,7 +37,6 @@ impl IndexActor { receiver, store, update_handler, - processing: RwLock::new(None), }) } @@ -174,9 +172,7 @@ impl IndexActor { .map_err(|e| IndexError::Error(e.into())) }; - *self.processing.write().await = Some(uuid); let result = get_result().await; - *self.processing.write().await = None; result } @@ -330,16 +326,13 @@ impl IndexActor { .await? .ok_or(IndexError::UnexistingIndex)?; - let processing = self.processing.read().await; - let is_indexing = *processing == Some(uuid); - spawn_blocking(move || { let rtxn = index.read_txn()?; Ok(IndexStats { size: index.size(), number_of_documents: index.number_of_documents(&rtxn)?, - is_indexing, + is_indexing: None, fields_distribution: index.fields_distribution(&rtxn)?, }) }) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 9d9ed02c1..2e93a2f7e 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -1,9 +1,11 @@ +use std::collections::BTreeMap; use std::path::Path; use std::sync::Arc; use std::time::Duration; use actix_web::web::{Bytes, Payload}; use anyhow::bail; +use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use log::info; use milli::update::{IndexDocumentsMethod, UpdateFormat}; @@ -37,6 +39,8 @@ pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { + #[serde(skip)] + pub uuid: Uuid, pub uid: String, name: String, #[serde(flatten)] @@ -63,11 +67,12 @@ pub struct IndexSettings { pub primary_key: Option, } -#[derive(Clone, Debug)] +#[derive(Serialize)] pub struct IndexStats { + #[serde(skip)] pub size: u64, pub number_of_documents: u64, - pub is_indexing: bool, + pub is_indexing: Option, pub fields_distribution: FieldsDistribution, } @@ -77,6 +82,13 @@ pub struct IndexController { update_handle: update_actor::UpdateActorHandleImpl, } +#[derive(Serialize)] +pub struct Stats { + pub database_size: u64, + pub last_update: Option>, + pub indexes: BTreeMap, +} + impl IndexController { pub fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { let index_size = options.max_mdb_size.get_bytes() as usize; @@ -166,6 +178,7 @@ impl IndexController { Err(UuidError::UnexistingIndex(name)) => { let uuid = Uuid::new_v4(); let status = perform_update(uuid).await?; + self.index_handle.create_index(uuid, None).await?; self.uuid_resolver.insert(name, uuid).await?; Ok(status) } @@ -218,6 +231,7 @@ impl IndexController { Err(UuidError::UnexistingIndex(name)) if create => { let uuid = Uuid::new_v4(); let status = perform_udpate(uuid).await?; + self.index_handle.create_index(uuid, None).await?; self.uuid_resolver.insert(name, uuid).await?; Ok(status) } @@ -234,6 +248,7 @@ impl IndexController { let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; let meta = IndexMetadata { + uuid, name: uid.clone(), uid, meta, @@ -269,6 +284,7 @@ impl IndexController { for (uid, uuid) in uuids { let meta = self.index_handle.get_index_meta(uuid).await?; let meta = IndexMetadata { + uuid, name: uid.clone(), uid, meta, @@ -326,6 +342,7 @@ impl IndexController { let uuid = self.uuid_resolver.get(uid.clone()).await?; let meta = self.index_handle.update_index(uuid, index_settings).await?; let meta = IndexMetadata { + uuid, name: uid.clone(), uid, meta, @@ -343,6 +360,7 @@ impl IndexController { let uuid = self.uuid_resolver.get(uid.clone()).await?; let meta = self.index_handle.get_index_meta(uuid).await?; let meta = IndexMetadata { + uuid, name: uid.clone(), uid, meta, @@ -350,19 +368,43 @@ impl IndexController { Ok(meta) } - pub async fn get_stats(&self, uid: String) -> anyhow::Result { - let uuid = self.uuid_resolver.get(uid.clone()).await?; - - Ok(self.index_handle.get_index_stats(uuid).await?) - } - - pub async fn get_updates_size(&self) -> anyhow::Result { - Ok(self.update_handle.get_size().await?) - } - pub async fn get_uuids_size(&self) -> anyhow::Result { Ok(self.uuid_resolver.get_size().await?) } + + pub async fn get_index_stats(&self, uid: String) -> anyhow::Result { + let uuid = self.uuid_resolver.get(uid).await?; + let update_infos = self.update_handle.get_info().await?; + let mut stats = self.index_handle.get_index_stats(uuid).await?; + stats.is_indexing = (Some(uuid) == update_infos.processing).into(); + Ok(stats) + } + + pub async fn get_all_stats(&self) -> anyhow::Result { + let update_infos = self.update_handle.get_info().await?; + let mut database_size = self.get_uuids_size().await? + update_infos.size; + let mut last_update: Option> = None; + let mut indexes = BTreeMap::new(); + + for index in self.list_indexes().await? { + let mut index_stats = self.index_handle.get_index_stats(index.uuid).await?; + database_size += index_stats.size; + + last_update = last_update.map_or(Some(index.meta.updated_at), |last| { + Some(last.max(index.meta.updated_at)) + }); + + index_stats.is_indexing = (Some(index.uuid) == update_infos.processing).into(); + + indexes.insert(index.uid, index_stats); + } + + Ok(Stats { + database_size, + last_update, + indexes, + }) + } } pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 2a752d0b3..0b5e88270 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -10,7 +10,7 @@ use tokio::sync::mpsc; use uuid::Uuid; use futures::StreamExt; -use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore}; +use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore, UpdateStoreInfo}; use crate::index_controller::index_actor::{IndexActorHandle, CONCURRENT_INDEX_MSG}; use crate::index_controller::{UpdateMeta, UpdateStatus}; @@ -81,8 +81,8 @@ where Some(Snapshot { uuids, path, ret }) => { let _ = ret.send(self.handle_snapshot(uuids, path).await); } - Some(GetSize { ret }) => { - let _ = ret.send(self.handle_get_size().await); + Some(GetInfo { ret }) => { + let _ = ret.send(self.handle_get_info().await); } None => break, } @@ -232,17 +232,27 @@ where Ok(()) } - async fn handle_get_size(&self) -> Result { + async fn handle_get_info(&self) -> Result { let update_store = self.store.clone(); - let size = tokio::task::spawn_blocking(move || -> anyhow::Result { + let processing = self.store.processing.clone(); + let info = tokio::task::spawn_blocking(move || -> anyhow::Result { let txn = update_store.env.read_txn()?; - - update_store.get_size(&txn) + let size = update_store.get_size(&txn)?; + let processing = processing + .read() + .as_ref() + .map(|(uuid, _)| uuid) + .cloned(); + let info = UpdateStoreInfo { + size, processing + }; + Ok(info) }) .await .map_err(|e| UpdateError::Error(e.into()))? .map_err(|e| UpdateError::Error(e.into()))?; - Ok(size) + + Ok(info) } } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 5c5a3e051..f79ef0e4e 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::index_controller::IndexActorHandle; use super::{ - PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStatus, + PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStatus, UpdateStoreInfo }; #[derive(Clone)] @@ -70,9 +70,9 @@ where receiver.await.expect("update actor killed.") } - async fn get_size(&self) -> Result { + async fn get_info(&self) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::GetSize { ret }; + let msg = UpdateMsg::GetInfo { ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 0f0005862..6082ad280 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use super::{PayloadData, Result, UpdateMeta, UpdateStatus}; +use super::{PayloadData, Result, UpdateMeta, UpdateStatus, UpdateStoreInfo}; pub enum UpdateMsg { Update { @@ -30,7 +30,7 @@ pub enum UpdateMsg { path: PathBuf, ret: oneshot::Sender>, }, - GetSize { - ret: oneshot::Sender>, + GetInfo { + ret: oneshot::Sender>, }, } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 0f815dbf3..faeb140a6 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -32,6 +32,14 @@ pub enum UpdateError { UnexistingUpdate(u64), } +pub struct UpdateStoreInfo { + /// Size of the update store in bytes. + pub size: u64, + /// Uuid of the currently processing update if it exists + pub processing: Option, + +} + #[async_trait::async_trait] #[cfg_attr(test, automock(type Data=Vec;))] pub trait UpdateActorHandle { @@ -41,7 +49,7 @@ pub trait UpdateActorHandle { async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()>; - async fn get_size(&self) -> Result; + async fn get_info(&self) -> Result; async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 79693ba1f..2e5350193 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -67,7 +67,7 @@ pub struct UpdateStore { processed_meta: Database>>, failed_meta: Database>>, aborted_meta: Database>>, - processing: Arc)>>>, + pub processing: Arc)>>>, notification_sender: mpsc::Sender<()>, /// A lock on the update loop. This is meant to prevent a snapshot to occur while an update is /// processing, while not preventing writes all together during an update diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index 226b62fcd..f2d1ddecc 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,15 +1,10 @@ -use std::collections::BTreeMap; - use actix_web::get; use actix_web::web; use actix_web::HttpResponse; -use chrono::{DateTime, Utc}; use serde::Serialize; -use crate::data::Stats; use crate::error::ResponseError; use crate::helpers::Authentication; -use crate::index_controller::IndexStats; use crate::routes::IndexParam; use crate::Data; @@ -19,59 +14,19 @@ pub fn services(cfg: &mut web::ServiceConfig) { .service(get_version); } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct IndexStatsResponse { - number_of_documents: u64, - is_indexing: bool, - fields_distribution: BTreeMap, -} - -impl From for IndexStatsResponse { - fn from(stats: IndexStats) -> Self { - Self { - number_of_documents: stats.number_of_documents, - is_indexing: stats.is_indexing, - fields_distribution: stats.fields_distribution.into_iter().collect(), - } - } -} - #[get("/indexes/{index_uid}/stats", wrap = "Authentication::Private")] async fn get_index_stats( data: web::Data, path: web::Path, ) -> Result { - let response: IndexStatsResponse = data.get_index_stats(path.index_uid.clone()).await?.into(); + let response = data.get_index_stats(path.index_uid.clone()).await?; Ok(HttpResponse::Ok().json(response)) } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct StatsResponse { - database_size: u64, - last_update: Option>, - indexes: BTreeMap, -} - -impl From for StatsResponse { - fn from(stats: Stats) -> Self { - Self { - database_size: stats.database_size, - last_update: stats.last_update, - indexes: stats - .indexes - .into_iter() - .map(|(uid, index_stats)| (uid, index_stats.into())) - .collect(), - } - } -} - #[get("/stats", wrap = "Authentication::Private")] async fn get_stats(data: web::Data) -> Result { - let response: StatsResponse = data.get_stats().await?.into(); + let response = data.get_all_stats().await?; Ok(HttpResponse::Ok().json(response)) } From c78f351300fa4d82604f3e7d3effb143b99fbabf Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 15 Apr 2021 19:54:25 +0200 Subject: [PATCH 292/522] fix tests --- .../src/index_controller/index_actor/mod.rs | 66 +++++++++++++++++++ meilisearch-http/src/index_controller/mod.rs | 2 + .../src/index_controller/snapshot.rs | 21 ++++-- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index e44891b5a..26f7a23db 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -1,3 +1,5 @@ +#[cfg(test)] +use std::sync::Arc; use std::path::PathBuf; use chrono::{DateTime, Utc}; @@ -70,6 +72,70 @@ pub enum IndexError { ExistingPrimaryKey, } +#[cfg(test)] +#[async_trait::async_trait] +impl IndexActorHandle for Arc { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + self.as_ref().create_index(uuid, primary_key).await + } + + async fn update( + &self, + uuid: Uuid, + meta: Processing, + data: std::fs::File, + ) -> anyhow::Result { + self.as_ref().update(uuid, meta, data).await + } + + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { + self.as_ref().search(uuid, query).await + } + + async fn settings(&self, uuid: Uuid) -> Result { + self.as_ref().settings(uuid).await + } + + async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + self.as_ref().documents(uuid, offset, limit, attributes_to_retrieve).await + } + + async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + self.as_ref().document(uuid, doc_id, attributes_to_retrieve).await + } + + async fn delete(&self, uuid: Uuid) -> Result<()> { + self.as_ref().delete(uuid).await + } + + async fn get_index_meta(&self, uuid: Uuid) -> Result { + self.as_ref().get_index_meta(uuid).await + } + + async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { + self.as_ref().update_index(uuid, index_settings).await + } + + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + self.as_ref().snapshot(uuid, path).await + } + + async fn get_index_stats(&self, uuid: Uuid) -> Result { + self.as_ref().get_index_stats(uuid).await + } +} + #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait IndexActorHandle { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 2e93a2f7e..35b8b1ecf 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -68,6 +68,7 @@ pub struct IndexSettings { } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] pub struct IndexStats { #[serde(skip)] pub size: u64, @@ -83,6 +84,7 @@ pub struct IndexController { } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] pub struct Stats { pub database_size: u64, pub last_update: Option>, diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index eba5a5f3b..40c72b0be 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -71,11 +71,9 @@ where return Ok(()); } - self.update_handle .snapshot(uuids, temp_snapshot_path.clone()) .await?; - let snapshot_dir = self.snapshot_path.clone(); let snapshot_path = self .snapshot_path @@ -133,19 +131,22 @@ pub fn load_snapshot( #[cfg(test)] mod test { + use std::sync::Arc; + use futures::future::{err, ok}; use rand::Rng; use tokio::time::timeout; use uuid::Uuid; use super::*; - use crate::index_controller::update_actor::{MockUpdateActorHandle, UpdateError}; + use crate::index_controller::update_actor::{UpdateError, MockUpdateActorHandle, UpdateActorHandleImpl}; + use crate::index_controller::index_actor::MockIndexActorHandle; use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidError}; #[actix_rt::test] async fn test_normal() { let mut rng = rand::thread_rng(); - let uuids_num = rng.gen_range(5, 10); + let uuids_num: usize = rng.gen_range(5, 10); let uuids = (0..uuids_num).map(|_| Uuid::new_v4()).collect::>(); let mut uuid_resolver = MockUuidResolverHandle::new(); @@ -155,13 +156,19 @@ mod test { .times(1) .returning(move |_| Box::pin(ok(uuids_clone.clone()))); - let mut update_handle = MockUpdateActorHandle::new(); let uuids_clone = uuids.clone(); - update_handle + let mut index_handle = MockIndexActorHandle::new(); + index_handle .expect_snapshot() .withf(move |uuid, _path| uuids_clone.contains(uuid)) .times(uuids_num) - .returning(move |_, _| Box::pin(ok(()))); + .returning(move |_, _| { + Box::pin(ok(())) + }); + + let dir = tempfile::tempdir_in(".").unwrap(); + let handle = Arc::new(index_handle); + let update_handle = UpdateActorHandleImpl::>::new(handle.clone(), dir.path(), 4096 * 100).unwrap(); let snapshot_path = tempfile::tempdir_in(".").unwrap(); let snapshot_service = SnapshotService::new( From 51829ad85e1ce028b6dde383b8e69c1e8138b8b5 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 19 Apr 2021 09:47:43 +0200 Subject: [PATCH 293/522] review fixes --- meilisearch-http/src/data/mod.rs | 1 - .../src/index_controller/index_actor/actor.rs | 22 +++++++------------ .../src/index_controller/update_actor/mod.rs | 1 - .../update_actor/update_store.rs | 13 +++++------ 4 files changed, 14 insertions(+), 23 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 60fb5ae20..e63bf9380 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -112,7 +112,6 @@ impl Data { Ok(self.index_controller.get_all_stats().await?) } - #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index ff7064600..e9e55606f 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -159,22 +159,16 @@ impl IndexActor { meta: Processing, data: File, ) -> Result { - let get_result = || async { - debug!("Processing update {}", meta.id()); - let update_handler = self.update_handler.clone(); - let index = match self.store.get(uuid).await? { - Some(index) => index, - None => self.store.create(uuid, None).await?, - }; - - spawn_blocking(move || update_handler.handle_update(meta, data, index)) - .await - .map_err(|e| IndexError::Error(e.into())) + debug!("Processing update {}", meta.id()); + let update_handler = self.update_handler.clone(); + let index = match self.store.get(uuid).await? { + Some(index) => index, + None => self.store.create(uuid, None).await?, }; - let result = get_result().await; - - result + spawn_blocking(move || update_handler.handle_update(meta, data, index)) + .await + .map_err(|e| IndexError::Error(e.into())) } async fn handle_settings(&self, uuid: Uuid) -> Result { diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index faeb140a6..6b7ed7b9b 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -37,7 +37,6 @@ pub struct UpdateStoreInfo { pub size: u64, /// Uuid of the currently processing update if it exists pub processing: Option, - } #[async_trait::async_trait] diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 2e5350193..70f20e901 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -130,7 +130,7 @@ where // Init update loop to perform any pending updates at launch. // Since we just launched the update store, and we still own the receiving end of the - // channel, this call is guarenteed to succeed. + // channel, this call is guaranteed to succeed. notification_sender.try_send(()).expect("Failed to init update store"); let update_store = Arc::new(UpdateStore { @@ -339,20 +339,19 @@ where let pending = self .pending_meta .prefix_iter(&rtxn, index_uuid.as_bytes())? - .filter_map(Result::ok) - .filter_map(|(_, p)| { + .filter_map(|entry| { + let (_, p) = entry.ok()?; if let Some((uuid, ref processing)) = *processing { // Filter out the currently processing update if it is from this index. if uuid == index_uuid && processing.id() == p.id() { None } else { - Some(p) + Some(UpdateStatus::from(p)) } } else { - Some(p) + Some(UpdateStatus::from(p)) } - }) - .map(UpdateStatus::from); + }); updates.extend(pending); From 4fe2a13c713314537fbfc484067e175f9b420b69 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 22 Apr 2021 10:14:29 +0200 Subject: [PATCH 294/522] rewrite update store --- Cargo.lock | 33 +- meilisearch-http/Cargo.toml | 4 +- meilisearch-http/src/index/mod.rs | 6 +- meilisearch-http/src/index/updates.rs | 36 +- .../src/index_controller/index_actor/actor.rs | 35 +- .../index_actor/handle_impl.rs | 46 +- .../index_controller/index_actor/message.rs | 28 +- .../src/index_controller/index_actor/mod.rs | 194 ++-- .../src/index_controller/index_actor/store.rs | 18 +- meilisearch-http/src/index_controller/mod.rs | 42 +- .../src/index_controller/snapshot.rs | 20 +- .../index_controller/update_actor/actor.rs | 125 ++- .../update_actor/handle_impl.rs | 7 +- .../index_controller/update_actor/message.rs | 3 +- .../src/index_controller/update_actor/mod.rs | 15 +- .../update_actor/update_store.rs | 905 ++++++++++-------- .../src/index_controller/update_handler.rs | 9 +- .../src/index_controller/updates.rs | 138 ++- .../index_controller/uuid_resolver/actor.rs | 4 +- .../uuid_resolver/handle_impl.rs | 3 +- .../index_controller/uuid_resolver/message.rs | 3 +- .../src/index_controller/uuid_resolver/mod.rs | 3 +- .../index_controller/uuid_resolver/store.rs | 11 +- meilisearch-http/src/routes/document.rs | 7 +- meilisearch-http/tests/common/index.rs | 5 +- .../tests/documents/add_documents.rs | 12 +- meilisearch-http/tests/settings/distinct.rs | 8 +- meilisearch-http/tests/settings/mod.rs | 2 +- 28 files changed, 896 insertions(+), 826 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25acf9bda..6cd5e26a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,6 +286,12 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" +[[package]] +name = "arc-swap" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73" + [[package]] name = "assert-json-diff" version = "1.0.1" @@ -295,19 +301,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "async-compression" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72c1f1154e234325b50864a349b9c8e56939e266a4c307c0f159812df2f9537" -dependencies = [ - "flate2", - "futures-core", - "memchr", - "pin-project-lite 0.2.6", - "tokio 0.2.25", -] - [[package]] name = "async-stream" version = "0.3.1" @@ -775,16 +768,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "dashmap" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" -dependencies = [ - "cfg-if 1.0.0", - "num_cpus", -] - [[package]] name = "debugid" version = "0.7.2" @@ -1751,17 +1734,15 @@ dependencies = [ "actix-web", "actix-web-static-files", "anyhow", + "arc-swap", "assert-json-diff", - "async-compression", "async-stream", "async-trait", "byte-unit", - "bytemuck", "bytes 0.6.0", "cargo_toml", "chrono", "crossbeam-channel", - "dashmap", "either", "env_logger 0.8.3", "flate2", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index c1eb6b8dc..e6912e428 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -28,15 +28,13 @@ actix-service = "2.0.0" actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] } actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true } anyhow = "1.0.36" -async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } async-stream = "0.3.0" async-trait = "0.1.42" +arc-swap = "1.2.0" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } -bytemuck = "1.5.1" bytes = "0.6.0" chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.0" -dashmap = "4.0.2" either = "1.6.1" env_logger = "0.8.2" flate2 = "1.0.19" diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 0bb20d3ed..43c8b0193 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; use crate::helpers::EnvSizer; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; -pub use updates::{Facets, Settings, UpdateResult}; +pub use updates::{Facets, Settings}; mod search; mod updates; @@ -59,9 +59,7 @@ impl Index { }) .transpose()? .unwrap_or_else(BTreeSet::new); - let distinct_attribute = self - .distinct_attribute(&txn)? - .map(String::from); + let distinct_attribute = self.distinct_attribute(&txn)?.map(String::from); Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index f91da257f..038f1f7e6 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -4,17 +4,11 @@ use std::num::NonZeroUsize; use flate2::read::GzDecoder; use log::info; -use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{de::Deserializer, Deserialize, Serialize}; use super::Index; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateResult { - DocumentsAddition(DocumentAdditionResult), - DocumentDeletion { deleted: u64 }, - Other, -} +use crate::index_controller::UpdateResult; #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -91,7 +85,7 @@ impl Index { &self, format: UpdateFormat, method: IndexDocumentsMethod, - content: impl io::Read, + content: Option, update_builder: UpdateBuilder, primary_key: Option<&str>, ) -> anyhow::Result { @@ -108,16 +102,15 @@ impl Index { builder.update_format(format); builder.index_documents_method(method); - let gzipped = false; - let reader = if gzipped { - Box::new(GzDecoder::new(content)) - } else { - Box::new(content) as Box - }; + let indexing_callback = + |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); - let result = builder.execute(reader, |indexing_step, update_id| { - info!("update {}: {:?}", update_id, indexing_step) - }); + let gzipped = false; + let result = match content { + Some(content) if gzipped => builder.execute(GzDecoder::new(content), indexing_callback), + Some(content) => builder.execute(content, indexing_callback), + None => builder.execute(std::io::empty(), indexing_callback), + }; info!("document addition done: {:?}", result); @@ -228,10 +221,13 @@ impl Index { pub fn delete_documents( &self, - document_ids: impl io::Read, + document_ids: Option, update_builder: UpdateBuilder, ) -> anyhow::Result { - let ids: Vec = serde_json::from_reader(document_ids)?; + let ids = match document_ids { + Some(reader) => serde_json::from_reader(reader)?, + None => Vec::::new(), + }; let mut txn = self.write_txn()?; let mut builder = update_builder.delete_documents(&mut txn, self)?; diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index e9e55606f..05130e795 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -11,13 +11,13 @@ use tokio::task::spawn_blocking; use uuid::Uuid; use crate::index::{Document, SearchQuery, SearchResult, Settings}; -use crate::index_controller::update_handler::UpdateHandler; use crate::index_controller::{ - get_arc_ownership_blocking, updates::Processing, IndexStats, UpdateMeta, + get_arc_ownership_blocking, update_handler::UpdateHandler, Failed, IndexStats, Processed, + Processing, }; use crate::option::IndexerOpts; -use super::{IndexError, IndexMeta, IndexMsg, IndexSettings, IndexStore, Result, UpdateResult}; +use super::{IndexError, IndexMeta, IndexMsg, IndexResult, IndexSettings, IndexStore}; pub const CONCURRENT_INDEX_MSG: usize = 10; @@ -28,7 +28,7 @@ pub struct IndexActor { } impl IndexActor { - pub fn new(receiver: mpsc::Receiver, store: S) -> Result { + pub fn new(receiver: mpsc::Receiver, store: S) -> IndexResult { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; let update_handler = Arc::new(update_handler); @@ -40,9 +40,6 @@ impl IndexActor { }) } - /// `run` poll the write_receiver and read_receiver concurrently, but while messages send - /// through the read channel are processed concurrently, the messages sent through the write - /// channel are processed one at a time. pub async fn run(mut self) { let mut receiver = self .receiver @@ -145,7 +142,7 @@ impl IndexActor { &self, uuid: Uuid, primary_key: Option, - ) -> Result { + ) -> IndexResult { let index = self.store.create(uuid, primary_key).await?; let meta = spawn_blocking(move || IndexMeta::new(&index)) .await @@ -156,9 +153,9 @@ impl IndexActor { async fn handle_update( &self, uuid: Uuid, - meta: Processing, - data: File, - ) -> Result { + meta: Processing, + data: Option, + ) -> IndexResult> { debug!("Processing update {}", meta.id()); let update_handler = self.update_handler.clone(); let index = match self.store.get(uuid).await? { @@ -171,7 +168,7 @@ impl IndexActor { .map_err(|e| IndexError::Error(e.into())) } - async fn handle_settings(&self, uuid: Uuid) -> Result { + async fn handle_settings(&self, uuid: Uuid) -> IndexResult { let index = self .store .get(uuid) @@ -188,7 +185,7 @@ impl IndexActor { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result> { + ) -> IndexResult> { let index = self .store .get(uuid) @@ -208,7 +205,7 @@ impl IndexActor { uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ) -> Result { + ) -> IndexResult { let index = self .store .get(uuid) @@ -223,7 +220,7 @@ impl IndexActor { .map_err(|e| IndexError::Error(e.into()))? } - async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + async fn handle_delete(&self, uuid: Uuid) -> IndexResult<()> { let index = self.store.delete(uuid).await?; if let Some(index) = index { @@ -240,7 +237,7 @@ impl IndexActor { Ok(()) } - async fn handle_get_meta(&self, uuid: Uuid) -> Result { + async fn handle_get_meta(&self, uuid: Uuid) -> IndexResult { match self.store.get(uuid).await? { Some(index) => { let meta = spawn_blocking(move || IndexMeta::new(&index)) @@ -256,7 +253,7 @@ impl IndexActor { &self, uuid: Uuid, index_settings: IndexSettings, - ) -> Result { + ) -> IndexResult { let index = self .store .get(uuid) @@ -283,7 +280,7 @@ impl IndexActor { .map_err(|e| IndexError::Error(e.into()))? } - async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> { + async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> IndexResult<()> { use tokio::fs::create_dir_all; path.push("indexes"); @@ -313,7 +310,7 @@ impl IndexActor { Ok(()) } - async fn handle_get_stats(&self, uuid: Uuid) -> Result { + async fn handle_get_stats(&self, uuid: Uuid) -> IndexResult { let index = self .store .get(uuid) diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index d23357d38..719e12cee 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -3,14 +3,14 @@ use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use crate::index::{Document, SearchQuery, SearchResult, Settings}; -use crate::index_controller::{updates::Processing, UpdateMeta}; -use crate::index_controller::{IndexSettings, IndexStats}; - -use super::{ - IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore, Result, UpdateResult, +use crate::index_controller::{IndexSettings, IndexStats, Processing}; +use crate::{ + index::{Document, SearchQuery, SearchResult, Settings}, + index_controller::{Failed, Processed}, }; +use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, IndexResult, MapIndexStore}; + #[derive(Clone)] pub struct IndexActorHandleImpl { sender: mpsc::Sender, @@ -18,7 +18,11 @@ pub struct IndexActorHandleImpl { #[async_trait::async_trait] impl IndexActorHandle for IndexActorHandleImpl { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + async fn create_index( + &self, + uuid: Uuid, + primary_key: Option, + ) -> IndexResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { ret, @@ -32,9 +36,9 @@ impl IndexActorHandle for IndexActorHandleImpl { async fn update( &self, uuid: Uuid, - meta: Processing, - data: std::fs::File, - ) -> anyhow::Result { + meta: Processing, + data: Option, + ) -> anyhow::Result> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Update { ret, @@ -46,14 +50,14 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { + async fn search(&self, uuid: Uuid, query: SearchQuery) -> IndexResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Search { uuid, query, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn settings(&self, uuid: Uuid) -> Result { + async fn settings(&self, uuid: Uuid) -> IndexResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Settings { uuid, ret }; let _ = self.sender.send(msg).await; @@ -66,7 +70,7 @@ impl IndexActorHandle for IndexActorHandleImpl { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result> { + ) -> IndexResult> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Documents { uuid, @@ -84,7 +88,7 @@ impl IndexActorHandle for IndexActorHandleImpl { uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ) -> Result { + ) -> IndexResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Document { uuid, @@ -96,21 +100,25 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn delete(&self, uuid: Uuid) -> Result<()> { + async fn delete(&self, uuid: Uuid) -> IndexResult<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Delete { uuid, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn get_index_meta(&self, uuid: Uuid) -> Result { + async fn get_index_meta(&self, uuid: Uuid) -> IndexResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetMeta { uuid, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { + async fn update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> IndexResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::UpdateIndex { uuid, @@ -121,14 +129,14 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Snapshot { uuid, path, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn get_index_stats(&self, uuid: Uuid) -> Result { + async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetStats { uuid, ret }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index cb28f2868..d728b2564 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -4,21 +4,21 @@ use tokio::sync::oneshot; use uuid::Uuid; use crate::index::{Document, SearchQuery, SearchResult, Settings}; -use crate::index_controller::{updates::Processing, IndexStats, UpdateMeta}; +use crate::index_controller::{Failed, IndexStats, Processed, Processing}; -use super::{IndexMeta, IndexSettings, Result, UpdateResult}; +use super::{IndexMeta, IndexResult, IndexSettings}; pub enum IndexMsg { CreateIndex { uuid: Uuid, primary_key: Option, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, Update { uuid: Uuid, - meta: Processing, - data: std::fs::File, - ret: oneshot::Sender>, + meta: Processing, + data: Option, + ret: oneshot::Sender>>, }, Search { uuid: Uuid, @@ -27,41 +27,41 @@ pub enum IndexMsg { }, Settings { uuid: Uuid, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, Documents { uuid: Uuid, attributes_to_retrieve: Option>, offset: usize, limit: usize, - ret: oneshot::Sender>>, + ret: oneshot::Sender>>, }, Document { uuid: Uuid, attributes_to_retrieve: Option>, doc_id: String, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, Delete { uuid: Uuid, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, GetMeta { uuid: Uuid, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, UpdateIndex { uuid: Uuid, index_settings: IndexSettings, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, Snapshot { uuid: Uuid, path: PathBuf, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, GetStats { uuid: Uuid, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, } diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 26f7a23db..d49923fa0 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -1,5 +1,4 @@ -#[cfg(test)] -use std::sync::Arc; +use std::fs::File; use std::path::PathBuf; use chrono::{DateTime, Utc}; @@ -15,12 +14,8 @@ pub use handle_impl::IndexActorHandleImpl; use message::IndexMsg; use store::{IndexStore, MapIndexStore}; -use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; -use crate::index_controller::{ - updates::{Failed, Processed, Processing}, - IndexStats, UpdateMeta, -}; +use crate::index_controller::{Failed, Processed, Processing, IndexStats}; use super::IndexSettings; @@ -29,8 +24,7 @@ mod handle_impl; mod message; mod store; -pub type Result = std::result::Result; -type UpdateResult = std::result::Result, Failed>; +pub type IndexResult = std::result::Result; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -41,12 +35,12 @@ pub struct IndexMeta { } impl IndexMeta { - fn new(index: &Index) -> Result { + fn new(index: &Index) -> IndexResult { let txn = index.read_txn()?; Self::new_txn(index, &txn) } - fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { + fn new_txn(index: &Index, txn: &heed::RoTxn) -> IndexResult { let created_at = index.created_at(&txn)?; let updated_at = index.updated_at(&txn)?; let primary_key = index.primary_key(&txn)?.map(String::from); @@ -72,82 +66,19 @@ pub enum IndexError { ExistingPrimaryKey, } -#[cfg(test)] -#[async_trait::async_trait] -impl IndexActorHandle for Arc { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { - self.as_ref().create_index(uuid, primary_key).await - } - - async fn update( - &self, - uuid: Uuid, - meta: Processing, - data: std::fs::File, - ) -> anyhow::Result { - self.as_ref().update(uuid, meta, data).await - } - - async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { - self.as_ref().search(uuid, query).await - } - - async fn settings(&self, uuid: Uuid) -> Result { - self.as_ref().settings(uuid).await - } - - async fn documents( - &self, - uuid: Uuid, - offset: usize, - limit: usize, - attributes_to_retrieve: Option>, - ) -> Result> { - self.as_ref().documents(uuid, offset, limit, attributes_to_retrieve).await - } - - async fn document( - &self, - uuid: Uuid, - doc_id: String, - attributes_to_retrieve: Option>, - ) -> Result { - self.as_ref().document(uuid, doc_id, attributes_to_retrieve).await - } - - async fn delete(&self, uuid: Uuid) -> Result<()> { - self.as_ref().delete(uuid).await - } - - async fn get_index_meta(&self, uuid: Uuid) -> Result { - self.as_ref().get_index_meta(uuid).await - } - - async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { - self.as_ref().update_index(uuid, index_settings).await - } - - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { - self.as_ref().snapshot(uuid, path).await - } - - async fn get_index_stats(&self, uuid: Uuid) -> Result { - self.as_ref().get_index_stats(uuid).await - } -} - #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait IndexActorHandle { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn create_index(&self, uuid: Uuid, primary_key: Option) + -> IndexResult; async fn update( &self, uuid: Uuid, - meta: Processing, - data: std::fs::File, - ) -> anyhow::Result; - async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result; - async fn settings(&self, uuid: Uuid) -> Result; + meta: Processing, + data: Option, + ) -> anyhow::Result>; + async fn search(&self, uuid: Uuid, query: SearchQuery) -> IndexResult; + async fn settings(&self, uuid: Uuid) -> IndexResult; async fn documents( &self, @@ -155,16 +86,103 @@ pub trait IndexActorHandle { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result>; + ) -> IndexResult>; async fn document( &self, uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ) -> Result; - async fn delete(&self, uuid: Uuid) -> Result<()>; - async fn get_index_meta(&self, uuid: Uuid) -> Result; - async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result; - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; - async fn get_index_stats(&self, uuid: Uuid) -> Result; + ) -> IndexResult; + async fn delete(&self, uuid: Uuid) -> IndexResult<()>; + async fn get_index_meta(&self, uuid: Uuid) -> IndexResult; + async fn update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> IndexResult; + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; + async fn get_index_stats(&self, uuid: Uuid) -> IndexResult; +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use super::*; + + #[async_trait::async_trait] + /// Useful for passing around an `Arc` in tests. + impl IndexActorHandle for Arc { + async fn create_index( + &self, + uuid: Uuid, + primary_key: Option, + ) -> IndexResult { + self.as_ref().create_index(uuid, primary_key).await + } + + async fn update( + &self, + uuid: Uuid, + meta: Processing, + data: Option, + ) -> anyhow::Result> { + self.as_ref().update(uuid, meta, data).await + } + + async fn search(&self, uuid: Uuid, query: SearchQuery) -> IndexResult { + self.as_ref().search(uuid, query).await + } + + async fn settings(&self, uuid: Uuid) -> IndexResult { + self.as_ref().settings(uuid).await + } + + async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> IndexResult> { + self.as_ref() + .documents(uuid, offset, limit, attributes_to_retrieve) + .await + } + + async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> IndexResult { + self.as_ref() + .document(uuid, doc_id, attributes_to_retrieve) + .await + } + + async fn delete(&self, uuid: Uuid) -> IndexResult<()> { + self.as_ref().delete(uuid).await + } + + async fn get_index_meta(&self, uuid: Uuid) -> IndexResult { + self.as_ref().get_index_meta(uuid).await + } + + async fn update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> IndexResult { + self.as_ref().update_index(uuid, index_settings).await + } + + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + self.as_ref().snapshot(uuid, path).await + } + + async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { + self.as_ref().get_index_stats(uuid).await + } + } } diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 6250f515e..44f076f2f 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -8,16 +8,16 @@ use tokio::sync::RwLock; use tokio::task::spawn_blocking; use uuid::Uuid; -use super::{IndexError, Result}; +use super::{IndexError, IndexResult}; use crate::index::Index; type AsyncMap = Arc>>; #[async_trait::async_trait] pub trait IndexStore { - async fn create(&self, uuid: Uuid, primary_key: Option) -> Result; - async fn get(&self, uuid: Uuid) -> Result>; - async fn delete(&self, uuid: Uuid) -> Result>; + async fn create(&self, uuid: Uuid, primary_key: Option) -> IndexResult; + async fn get(&self, uuid: Uuid) -> IndexResult>; + async fn delete(&self, uuid: Uuid) -> IndexResult>; } pub struct MapIndexStore { @@ -40,14 +40,14 @@ impl MapIndexStore { #[async_trait::async_trait] impl IndexStore for MapIndexStore { - async fn create(&self, uuid: Uuid, primary_key: Option) -> Result { + async fn create(&self, uuid: Uuid, primary_key: Option) -> IndexResult { let path = self.path.join(format!("index-{}", uuid)); if path.exists() { return Err(IndexError::IndexAlreadyExists); } let index_size = self.index_size; - let index = spawn_blocking(move || -> Result { + let index = spawn_blocking(move || -> IndexResult { let index = open_index(&path, index_size)?; if let Some(primary_key) = primary_key { let mut txn = index.write_txn()?; @@ -64,7 +64,7 @@ impl IndexStore for MapIndexStore { Ok(index) } - async fn get(&self, uuid: Uuid) -> Result> { + async fn get(&self, uuid: Uuid) -> IndexResult> { let guard = self.index_store.read().await; match guard.get(&uuid) { Some(index) => Ok(Some(index.clone())), @@ -86,7 +86,7 @@ impl IndexStore for MapIndexStore { } } - async fn delete(&self, uuid: Uuid) -> Result> { + async fn delete(&self, uuid: Uuid) -> IndexResult> { let db_path = self.path.join(format!("index-{}", uuid)); fs::remove_dir_all(db_path) .await @@ -96,7 +96,7 @@ impl IndexStore for MapIndexStore { } } -fn open_index(path: impl AsRef, size: usize) -> Result { +fn open_index(path: impl AsRef, size: usize) -> IndexResult { std::fs::create_dir_all(&path).map_err(|e| IndexError::Error(e.into()))?; let mut options = EnvOpenOptions::new(); options.map_size(size); diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 35b8b1ecf..81e6d0a5e 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -8,23 +8,19 @@ use anyhow::bail; use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use log::info; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; use milli::FieldsDistribution; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::time::sleep; use uuid::Uuid; +pub use updates::*; use index_actor::IndexActorHandle; -use snapshot::load_snapshot; -use snapshot::SnapshotService; +use snapshot::{SnapshotService, load_snapshot}; use update_actor::UpdateActorHandle; -pub use updates::{Failed, Processed, Processing}; -use uuid_resolver::UuidError; -use uuid_resolver::UuidResolverHandle; +use uuid_resolver::{UuidError, UuidResolverHandle}; -use crate::index::{Document, SearchQuery, SearchResult}; -use crate::index::{Facets, Settings, UpdateResult}; +use crate::index::{Settings, Document, SearchQuery, SearchResult}; use crate::option::Opt; mod index_actor; @@ -34,8 +30,6 @@ mod update_handler; mod updates; mod uuid_resolver; -pub type UpdateStatus = updates::UpdateStatus; - #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { @@ -47,20 +41,6 @@ pub struct IndexMetadata { pub meta: index_actor::IndexMeta, } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum UpdateMeta { - DocumentsAddition { - method: IndexDocumentsMethod, - format: UpdateFormat, - primary_key: Option, - }, - ClearDocuments, - DeleteDocuments, - Settings(Settings), - Facets(Facets), -} - #[derive(Clone, Debug)] pub struct IndexSettings { pub uid: Option, @@ -73,6 +53,9 @@ pub struct IndexStats { #[serde(skip)] pub size: u64, pub number_of_documents: u64, + /// Whether the current index is performing an update. It is initially `None` when the + /// index returns it, since it is the `UpdateStore` that knows what index is currently indexing. It is + /// later set to either true or false, we we retrieve the information from the `UpdateStore` pub is_indexing: Option, pub fields_distribution: FieldsDistribution, } @@ -180,7 +163,8 @@ impl IndexController { Err(UuidError::UnexistingIndex(name)) => { let uuid = Uuid::new_v4(); let status = perform_update(uuid).await?; - self.index_handle.create_index(uuid, None).await?; + // ignore if index creation fails now, since it may already have been created + let _ = self.index_handle.create_index(uuid, None).await; self.uuid_resolver.insert(name, uuid).await?; Ok(status) } @@ -233,7 +217,8 @@ impl IndexController { Err(UuidError::UnexistingIndex(name)) if create => { let uuid = Uuid::new_v4(); let status = perform_udpate(uuid).await?; - self.index_handle.create_index(uuid, None).await?; + // ignore if index creation fails now, since it may already have been created + let _ = self.index_handle.create_index(uuid, None).await; self.uuid_resolver.insert(name, uuid).await?; Ok(status) } @@ -378,7 +363,8 @@ impl IndexController { let uuid = self.uuid_resolver.get(uid).await?; let update_infos = self.update_handle.get_info().await?; let mut stats = self.index_handle.get_index_stats(uuid).await?; - stats.is_indexing = (Some(uuid) == update_infos.processing).into(); + // Check if the currently indexing update is from out index. + stats.is_indexing = Some(Some(uuid) == update_infos.processing); Ok(stats) } @@ -396,7 +382,7 @@ impl IndexController { Some(last.max(index.meta.updated_at)) }); - index_stats.is_indexing = (Some(index.uuid) == update_infos.processing).into(); + index_stats.is_indexing = Some(Some(index.uuid) == update_infos.processing); indexes.insert(index.uid, index_stats); } diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 40c72b0be..2a456eb26 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -131,7 +131,8 @@ pub fn load_snapshot( #[cfg(test)] mod test { - use std::sync::Arc; + use std::iter::FromIterator; + use std::{collections::HashSet, sync::Arc}; use futures::future::{err, ok}; use rand::Rng; @@ -139,15 +140,19 @@ mod test { use uuid::Uuid; use super::*; - use crate::index_controller::update_actor::{UpdateError, MockUpdateActorHandle, UpdateActorHandleImpl}; use crate::index_controller::index_actor::MockIndexActorHandle; + use crate::index_controller::update_actor::{ + MockUpdateActorHandle, UpdateActorHandleImpl, UpdateError, + }; use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidError}; #[actix_rt::test] async fn test_normal() { let mut rng = rand::thread_rng(); let uuids_num: usize = rng.gen_range(5, 10); - let uuids = (0..uuids_num).map(|_| Uuid::new_v4()).collect::>(); + let uuids = (0..uuids_num) + .map(|_| Uuid::new_v4()) + .collect::>(); let mut uuid_resolver = MockUuidResolverHandle::new(); let uuids_clone = uuids.clone(); @@ -162,13 +167,12 @@ mod test { .expect_snapshot() .withf(move |uuid, _path| uuids_clone.contains(uuid)) .times(uuids_num) - .returning(move |_, _| { - Box::pin(ok(())) - }); + .returning(move |_, _| Box::pin(ok(()))); let dir = tempfile::tempdir_in(".").unwrap(); let handle = Arc::new(index_handle); - let update_handle = UpdateActorHandleImpl::>::new(handle.clone(), dir.path(), 4096 * 100).unwrap(); + let update_handle = + UpdateActorHandleImpl::>::new(handle.clone(), dir.path(), 4096 * 100).unwrap(); let snapshot_path = tempfile::tempdir_in(".").unwrap(); let snapshot_service = SnapshotService::new( @@ -214,7 +218,7 @@ mod test { uuid_resolver .expect_snapshot() .times(1) - .returning(move |_| Box::pin(ok(vec![uuid]))); + .returning(move |_| Box::pin(ok(HashSet::from_iter(Some(uuid))))); let mut update_handle = MockUpdateActorHandle::new(); update_handle diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 0b5e88270..c98df7d25 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -1,14 +1,16 @@ +use std::collections::HashSet; use std::io::SeekFrom; use std::path::{Path, PathBuf}; use std::sync::Arc; +use futures::StreamExt; use log::info; use oxidized_json_checker::JsonChecker; use tokio::fs; -use tokio::io::{AsyncSeekExt, AsyncWriteExt}; +use tokio::io::AsyncWriteExt; +use tokio::runtime::Handle; use tokio::sync::mpsc; use uuid::Uuid; -use futures::StreamExt; use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore, UpdateStoreInfo}; use crate::index_controller::index_actor::{IndexActorHandle, CONCURRENT_INDEX_MSG}; @@ -32,18 +34,14 @@ where path: impl AsRef, index_handle: I, ) -> anyhow::Result { - let path = path.as_ref().to_owned().join("updates"); + let path = path.as_ref().join("updates"); std::fs::create_dir_all(&path)?; let mut options = heed::EnvOpenOptions::new(); options.map_size(update_db_size); - let handle = index_handle.clone(); - let store = UpdateStore::open(options, &path, move |uuid, meta, file| { - futures::executor::block_on(handle.update(uuid, meta, file)) - }) - .map_err(|e| UpdateError::Error(e.into()))?; + let store = UpdateStore::open(options, &path, index_handle.clone())?; std::fs::create_dir_all(path.join("update_files"))?; assert!(path.exists()); Ok(Self { @@ -95,40 +93,54 @@ where meta: UpdateMeta, mut payload: mpsc::Receiver>, ) -> Result { - let update_file_id = uuid::Uuid::new_v4(); - let path = self - .path - .join(format!("update_files/update_{}", update_file_id)); - let mut file = fs::OpenOptions::new() - .read(true) - .write(true) - .create(true) - .open(&path) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; - while let Some(bytes) = payload.recv().await { - match bytes { - Ok(bytes) => { - file.write_all(bytes.as_ref()) + let file_path = match meta { + UpdateMeta::DocumentsAddition { .. } + | UpdateMeta::DeleteDocuments => { + + let update_file_id = uuid::Uuid::new_v4(); + let path = self + .path + .join(format!("update_files/update_{}", update_file_id)); + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&path) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))?; + + let mut file_len = 0; + while let Some(bytes) = payload.recv().await { + match bytes { + Ok(bytes) => { + file_len += bytes.as_ref().len(); + file.write_all(bytes.as_ref()) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))?; + } + Err(e) => { + return Err(UpdateError::Error(e)); + } + } + } + + if file_len != 0 { + file.flush() .await .map_err(|e| UpdateError::Error(Box::new(e)))?; - } - Err(e) => { - return Err(UpdateError::Error(e)); + let file = file.into_std().await; + Some((file, path)) + } else { + // empty update, delete the empty file. + fs::remove_file(&path) + .await + .map_err(|e| UpdateError::Error(Box::new(e)))?; + None } } - } - - file.flush() - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; - - file.seek(SeekFrom::Start(0)) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; - - let mut file = file.into_std().await; + _ => None + }; let update_store = self.store.clone(); @@ -136,12 +148,9 @@ where use std::io::{copy, sink, BufReader, Seek}; // If the payload is empty, ignore the check. - if file - .metadata() - .map_err(|e| UpdateError::Error(Box::new(e)))? - .len() - > 0 - { + let path = if let Some((mut file, path)) = file_path { + // set the file back to the beginning + file.seek(SeekFrom::Start(0)).map_err(|e| UpdateError::Error(Box::new(e)))?; // Check that the json payload is valid: let reader = BufReader::new(&mut file); let mut checker = JsonChecker::new(reader); @@ -153,7 +162,10 @@ where let _: serde_json::Value = serde_json::from_reader(file) .map_err(|e| UpdateError::Error(Box::new(e)))?; } - } + Some(path) + } else { + None + }; // The payload is valid, we can register it to the update store. update_store @@ -197,17 +209,11 @@ where Ok(()) } - async fn handle_snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { + async fn handle_snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - // acquire write lock to prevent further writes during snapshot - // the update lock must be acquired BEFORE the write lock to prevent dead lock - let _lock = update_store.update_lock.lock(); - let mut txn = update_store.env.write_txn()?; - - // create db snapshot - update_store.snapshot(&mut txn, &path)?; + update_store.snapshot(&uuids, &path)?; // Perform the snapshot of each index concurently. Only a third of the capabilities of // the index actor at a time not to put too much pressure on the index actor @@ -218,7 +224,7 @@ where .map(|&uuid| handle.snapshot(uuid, path.clone())) .buffer_unordered(CONCURRENT_INDEX_MSG / 3); - futures::executor::block_on(async { + Handle::current().block_on(async { while let Some(res) = stream.next().await { res?; } @@ -234,25 +240,14 @@ where async fn handle_get_info(&self) -> Result { let update_store = self.store.clone(); - let processing = self.store.processing.clone(); let info = tokio::task::spawn_blocking(move || -> anyhow::Result { - let txn = update_store.env.read_txn()?; - let size = update_store.get_size(&txn)?; - let processing = processing - .read() - .as_ref() - .map(|(uuid, _)| uuid) - .cloned(); - let info = UpdateStoreInfo { - size, processing - }; + let info = update_store.get_info()?; Ok(info) }) .await .map_err(|e| UpdateError::Error(e.into()))? .map_err(|e| UpdateError::Error(e.into()))?; - Ok(info) } } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index f79ef0e4e..999481573 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -1,12 +1,13 @@ +use std::collections::HashSet; use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use crate::index_controller::IndexActorHandle; +use crate::index_controller::{IndexActorHandle, UpdateStatus}; use super::{ - PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStatus, UpdateStoreInfo + PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStoreInfo, }; #[derive(Clone)] @@ -63,7 +64,7 @@ where receiver.await.expect("update actor killed.") } - async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()> { + async fn snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Snapshot { uuids, path, ret }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 6082ad280..17b2b3579 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::path::PathBuf; use tokio::sync::{mpsc, oneshot}; @@ -26,7 +27,7 @@ pub enum UpdateMsg { ret: oneshot::Sender>, }, Snapshot { - uuids: Vec, + uuids: HashSet, path: PathBuf, ret: oneshot::Sender>, }, diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 6b7ed7b9b..e7a12b7ff 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -3,22 +3,22 @@ mod handle_impl; mod message; mod update_store; -use std::path::PathBuf; +use std::{collections::HashSet, path::PathBuf}; use thiserror::Error; use tokio::sync::mpsc; use uuid::Uuid; -use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; use actor::UpdateActor; use message::UpdateMsg; +use update_store::UpdateStore; +pub use update_store::UpdateStoreInfo; pub use handle_impl::UpdateActorHandleImpl; pub type Result = std::result::Result; -type UpdateStore = update_store::UpdateStore; type PayloadData = std::result::Result>; #[cfg(test)] @@ -32,13 +32,6 @@ pub enum UpdateError { UnexistingUpdate(u64), } -pub struct UpdateStoreInfo { - /// Size of the update store in bytes. - pub size: u64, - /// Uuid of the currently processing update if it exists - pub processing: Option, -} - #[async_trait::async_trait] #[cfg_attr(test, automock(type Data=Vec;))] pub trait UpdateActorHandle { @@ -47,7 +40,7 @@ pub trait UpdateActorHandle { async fn get_all_updates_status(&self, uuid: Uuid) -> Result>; async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; - async fn snapshot(&self, uuids: Vec, path: PathBuf) -> Result<()>; + async fn snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()>; async fn get_info(&self) -> Result; async fn update( &self, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 70f20e901..795b6c36a 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -1,148 +1,213 @@ use std::borrow::Cow; +use std::collections::{BTreeMap, HashSet}; use std::convert::TryInto; use std::fs::{copy, create_dir_all, remove_file, File}; use std::mem::size_of; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; use anyhow::Context; -use bytemuck::{Pod, Zeroable}; -use heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; +use arc_swap::ArcSwap; +use heed::types::{ByteSlice, OwnedType, SerdeJson}; +use heed::zerocopy::U64; use heed::{BytesDecode, BytesEncode, CompactionOption, Database, Env, EnvOpenOptions}; -use parking_lot::{Mutex, RwLock}; -use serde::{Deserialize, Serialize}; +use parking_lot::{Mutex, MutexGuard}; +use tokio::runtime::Handle; use tokio::sync::mpsc; use uuid::Uuid; +use super::UpdateMeta; use crate::helpers::EnvSizer; -use crate::index_controller::updates::*; +use crate::index_controller::{IndexActorHandle, updates::*}; #[allow(clippy::upper_case_acronyms)] -type BEU64 = heed::zerocopy::U64; +type BEU64 = U64; -struct IndexUuidUpdateIdCodec; +struct NextIdCodec; -#[repr(C)] -#[derive(Copy, Clone)] -struct IndexUuidUpdateId(Uuid, BEU64); +enum NextIdKey { + Global, + Index(Uuid), +} -// Is Uuid really zeroable (semantically)? -unsafe impl Zeroable for IndexUuidUpdateId {} -unsafe impl Pod for IndexUuidUpdateId {} +pub struct UpdateStoreInfo { + /// Size of the update store in bytes. + pub size: u64, + /// Uuid of the currently processing update if it exists + pub processing: Option, +} -impl IndexUuidUpdateId { - fn new(uuid: Uuid, update_id: u64) -> Self { - Self(uuid, BEU64::new(update_id)) +/// A data structure that allows concurrent reads AND exactly one writer. +pub struct StateLock { + lock: Mutex<()>, + data: ArcSwap, +} + +struct StateLockGuard<'a> { + _lock: MutexGuard<'a, ()>, + state: &'a StateLock, +} + +impl StateLockGuard<'_> { + fn swap(&self, state: State) -> Arc { + self.state.data.swap(Arc::new(state)) } } -const UUID_SIZE: usize = size_of::(); -const U64_SIZE: usize = size_of::(); +impl StateLock { + fn from_state(state: State) -> Self { + let lock = Mutex::new(()); + let data = ArcSwap::from(Arc::new(state)); + Self { data, lock } + } -impl<'a> BytesEncode<'a> for IndexUuidUpdateIdCodec { - type EItem = IndexUuidUpdateId; + fn read(&self) -> Arc { + self.data.load().clone() + } - fn bytes_encode(item: &'a Self::EItem) -> Option> { - let bytes = bytemuck::cast_ref::(item); - Some(Cow::Borrowed(&bytes[..])) + fn write(&self) -> StateLockGuard { + let _lock = self.lock.lock(); + let state = &self; + StateLockGuard { state, _lock } } } -impl<'a> BytesDecode<'a> for IndexUuidUpdateIdCodec { +pub enum State { + Idle, + Processing(Uuid, Processing), + Snapshoting, +} + +impl<'a> BytesEncode<'a> for NextIdCodec { + type EItem = NextIdKey; + + fn bytes_encode(item: &'a Self::EItem) -> Option> { + match item { + NextIdKey::Global => Some(Cow::Borrowed(b"__global__")), + NextIdKey::Index(ref uuid) => Some(Cow::Borrowed(uuid.as_bytes())), + } + } +} + +struct PendingKeyCodec; + +impl<'a> BytesEncode<'a> for PendingKeyCodec { + type EItem = (u64, Uuid, u64); + + fn bytes_encode((global_id, uuid, update_id): &'a Self::EItem) -> Option> { + let mut bytes = Vec::with_capacity(size_of::()); + bytes.extend_from_slice(&global_id.to_be_bytes()); + bytes.extend_from_slice(uuid.as_bytes()); + bytes.extend_from_slice(&update_id.to_be_bytes()); + Some(Cow::Owned(bytes)) + } +} + +impl<'a> BytesDecode<'a> for PendingKeyCodec { + type DItem = (u64, Uuid, u64); + + fn bytes_decode(bytes: &'a [u8]) -> Option { + let global_id_bytes = bytes.get(0..size_of::())?.try_into().ok()?; + let global_id = u64::from_be_bytes(global_id_bytes); + + let uuid_bytes = bytes + .get(size_of::()..(size_of::() + size_of::()))? + .try_into() + .ok()?; + let uuid = Uuid::from_bytes(uuid_bytes); + + let update_id_bytes = bytes + .get((size_of::() + size_of::())..)? + .try_into() + .ok()?; + let update_id = u64::from_be_bytes(update_id_bytes); + + Some((global_id, uuid, update_id)) + } +} + +struct UpdateKeyCodec; + +impl<'a> BytesEncode<'a> for UpdateKeyCodec { + type EItem = (Uuid, u64); + + fn bytes_encode((uuid, update_id): &'a Self::EItem) -> Option> { + let mut bytes = Vec::with_capacity(size_of::()); + bytes.extend_from_slice(uuid.as_bytes()); + bytes.extend_from_slice(&update_id.to_be_bytes()); + Some(Cow::Owned(bytes)) + } +} + +impl<'a> BytesDecode<'a> for UpdateKeyCodec { type DItem = (Uuid, u64); fn bytes_decode(bytes: &'a [u8]) -> Option { - let bytes = bytes.try_into().ok()?; - let IndexUuidUpdateId(uuid, id) = - bytemuck::cast_ref::<[u8; UUID_SIZE + U64_SIZE], IndexUuidUpdateId>(bytes); - Some((*uuid, id.get())) + let uuid_bytes = bytes.get(0..size_of::())?.try_into().ok()?; + let uuid = Uuid::from_bytes(uuid_bytes); + + let update_id_bytes = bytes.get(size_of::()..)?.try_into().ok()?; + let update_id = u64::from_be_bytes(update_id_bytes); + + Some((uuid, update_id)) } } #[derive(Clone)] -pub struct UpdateStore { +pub struct UpdateStore { pub env: Env, - pending_meta: Database>>, - pending: Database>, - processed_meta: Database>>, - failed_meta: Database>>, - aborted_meta: Database>>, - pub processing: Arc)>>>, + /// A queue containing the updates to process, ordered by arrival. + /// The key are built as follow: + /// | global_update_id | index_uuid | update_id | + /// | 8-bytes | 16-bytes | 8-bytes | + pending_queue: Database>, + /// Map indexes to the next available update id. If NextIdKey::Global is queried, then the next + /// global update id is returned + next_update_id: Database>, + /// Contains all the performed updates meta, be they failed, aborted, or processed. + /// The keys are built as follow: + /// | Uuid | id | + /// | 16-bytes | 8-bytes | + updates: Database>, + /// Indicates the current state of the update store, + state: Arc, + /// Wake up the loop when a new event occurs. notification_sender: mpsc::Sender<()>, - /// A lock on the update loop. This is meant to prevent a snapshot to occur while an update is - /// processing, while not preventing writes all together during an update - pub update_lock: Arc>, } -pub trait HandleUpdate { - fn handle_update( - &mut self, - index_uuid: Uuid, - meta: Processing, - content: File, - ) -> anyhow::Result, Failed>>; -} - -impl HandleUpdate for F -where - F: FnMut(Uuid, Processing, File) -> anyhow::Result, Failed>>, -{ - fn handle_update( - &mut self, - index_uuid: Uuid, - meta: Processing, - content: File, - ) -> anyhow::Result, Failed>> { - self(index_uuid, meta, content) - } -} - -impl UpdateStore -where - M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, - N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, - E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, -{ - pub fn open( +impl UpdateStore { + pub fn open( mut options: EnvOpenOptions, - path: P, - update_handler: U, - ) -> anyhow::Result> - where - P: AsRef, - U: HandleUpdate + Sync + Clone + Send + 'static, - { + path: impl AsRef, + index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static, + ) -> anyhow::Result> { options.max_dbs(5); let env = options.open(path)?; - let pending_meta = env.create_database(Some("pending-meta"))?; - let pending = env.create_database(Some("pending"))?; - let processed_meta = env.create_database(Some("processed-meta"))?; - let aborted_meta = env.create_database(Some("aborted-meta"))?; - let failed_meta = env.create_database(Some("failed-meta"))?; - let processing = Arc::new(RwLock::new(None)); + let pending_queue = env.create_database(Some("pending-queue"))?; + let next_update_id = env.create_database(Some("next-update-id"))?; + let updates = env.create_database(Some("updates"))?; let (notification_sender, mut notification_receiver) = mpsc::channel(10); // Send a first notification to trigger the process. let _ = notification_sender.send(()); - let update_lock = Arc::new(Mutex::new(())); + let state = Arc::new(StateLock::from_state(State::Idle)); // Init update loop to perform any pending updates at launch. // Since we just launched the update store, and we still own the receiving end of the // channel, this call is guaranteed to succeed. - notification_sender.try_send(()).expect("Failed to init update store"); + notification_sender + .try_send(()) + .expect("Failed to init update store"); let update_store = Arc::new(UpdateStore { env, - pending, - pending_meta, - processed_meta, - aborted_meta, notification_sender, - failed_meta, - processing, - update_lock, + next_update_id, + pending_queue, + updates, + state, }); // We need a weak reference so we can take ownership on the arc later when we @@ -154,7 +219,7 @@ where loop { match update_store_weak.upgrade() { Some(update_store) => { - let handler = update_handler.clone(); + let handler = index_handle.clone(); let res = tokio::task::spawn_blocking(move || { update_store.process_pending_update(handler) }) @@ -176,75 +241,47 @@ where Ok(update_store) } - /// Returns the new biggest id to use to store the new update. - fn new_update_id(&self, txn: &heed::RoTxn, index_uuid: Uuid) -> heed::Result { - // TODO: this is a very inneficient process for finding the next update id for each index, - // and needs to be made better. - let last_pending = self - .pending_meta - .remap_data_type::() - .prefix_iter(txn, index_uuid.as_bytes())? - .remap_key_type::() - .last() - .transpose()? - .map(|((_, id), _)| id); + /// Returns the next global update id and the next update id for a given `index_uuid`. + fn next_update_id(&self, txn: &mut heed::RwTxn, index_uuid: Uuid) -> heed::Result<(u64, u64)> { + let global_id = self + .next_update_id + .get(txn, &NextIdKey::Global)? + .map(U64::get) + .unwrap_or_default(); + let update_id = self + .next_update_id + .get(txn, &NextIdKey::Index(index_uuid))? + .map(U64::get) + .unwrap_or_default(); - let last_processed = self - .processed_meta - .remap_data_type::() - .prefix_iter(txn, index_uuid.as_bytes())? - .remap_key_type::() - .last() - .transpose()? - .map(|((_, id), _)| id); + self.next_update_id + .put(txn, &NextIdKey::Global, &BEU64::new(global_id + 1))?; + self.next_update_id.put( + txn, + &NextIdKey::Index(index_uuid), + &BEU64::new(update_id + 1), + )?; - let last_aborted = self - .aborted_meta - .remap_data_type::() - .prefix_iter(txn, index_uuid.as_bytes())? - .remap_key_type::() - .last() - .transpose()? - .map(|((_, id), _)| id); - - let last_update_id = [last_pending, last_processed, last_aborted] - .iter() - .copied() - .flatten() - .max(); - - match last_update_id { - Some(last_id) => Ok(last_id + 1), - None => Ok(0), - } + Ok((global_id, update_id)) } /// Registers the update content in the pending store and the meta /// into the pending-meta store. Returns the new unique update id. pub fn register_update( &self, - meta: M, - content: impl AsRef, + meta: UpdateMeta, + content: Option>, index_uuid: Uuid, - ) -> heed::Result> { - let mut wtxn = self.env.write_txn()?; + ) -> heed::Result { + let mut txn = self.env.write_txn()?; - // We ask the update store to give us a new update id, this is safe, - // no other update can have the same id because we use a write txn before - // asking for the id and registering it so other update registering - // will be forced to wait for a new write txn. - let update_id = self.new_update_id(&wtxn, index_uuid)?; - let meta = Enqueued::new(meta, update_id); - let key = IndexUuidUpdateId::new(index_uuid, update_id); - self.pending_meta - .remap_key_type::() - .put(&mut wtxn, &key, &meta)?; + let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?; + let meta = Enqueued::new(meta, update_id, content.map(|p| p.as_ref().to_owned())); - self.pending - .remap_key_type::() - .put(&mut wtxn, &key, &content.as_ref().to_owned())?; + self.pending_queue + .put(&mut txn, &(global_id, index_uuid, update_id), &meta)?; - wtxn.commit()?; + txn.commit()?; self.notification_sender .blocking_send(()) @@ -255,68 +292,62 @@ where /// Executes the user provided function on the next pending update (the one with the lowest id). /// This is asynchronous as it let the user process the update with a read-only txn and /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, mut handler: U) -> anyhow::Result> - where - U: HandleUpdate, - { - let _lock = self.update_lock.lock(); + fn process_pending_update( + &self, + index_handle: impl IndexActorHandle, + ) -> anyhow::Result> { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; - - let first_meta = self - .pending_meta - .remap_key_type::() - .first(&rtxn)?; + let first_meta = self.pending_queue.first(&rtxn)?; + drop(rtxn); // If there is a pending update we process and only keep // a reader while processing it, not a writer. match first_meta { - Some(((index_uuid, update_id), pending)) => { - let key = IndexUuidUpdateId::new(index_uuid, update_id); - let content_path = self - .pending - .remap_key_type::() - .get(&rtxn, &key)? - .expect("associated update content"); - - // we change the state of the update from pending to processing before we pass it - // to the update handler. Processing store is non persistent to be able recover - // from a failure + Some(((global_id, index_uuid, update_id), mut pending)) => { + let content_path = pending.content.take(); let processing = pending.processing(); - self.processing - .write() - .replace((index_uuid, processing.clone())); - let file = File::open(&content_path) - .with_context(|| format!("file at path: {:?}", &content_path))?; + + // Acquire the state lock and set the current state to processing. + let state = self.state.write(); + state.swap(State::Processing(index_uuid, processing.clone())); + + let file = match content_path { + Some(ref path) => { + let file = File::open(path) + .with_context(|| format!("file at path: {:?}", &content_path))?; + Some(file) + } + None => None, + }; // Process the pending update using the provided user function. - let result = handler.handle_update(index_uuid, processing, file)?; - drop(rtxn); + let result = Handle::current() + .block_on(index_handle.update(index_uuid, processing, file))?; // Once the pending update have been successfully processed // we must remove the content from the pending and processing stores and // write the *new* meta to the processed-meta store and commit. let mut wtxn = self.env.write_txn()?; - self.processing.write().take(); - self.pending_meta - .remap_key_type::() - .delete(&mut wtxn, &key)?; + self.pending_queue + .delete(&mut wtxn, &(global_id, index_uuid, update_id))?; - remove_file(&content_path)?; - - self.pending - .remap_key_type::() - .delete(&mut wtxn, &key)?; - match result { - Ok(processed) => self - .processed_meta - .remap_key_type::() - .put(&mut wtxn, &key, &processed)?, - Err(failed) => self - .failed_meta - .remap_key_type::() - .put(&mut wtxn, &key, &failed)?, + if let Some(path) = content_path { + remove_file(&path)?; } + + let result = match result { + Ok(res) => res.into(), + Err(res) => res.into(), + }; + + self.updates.remap_key_type::().put( + &mut wtxn, + &(index_uuid, update_id), + &result, + )?; + wtxn.commit()?; + state.swap(State::Idle); Ok(Some(())) } @@ -324,245 +355,127 @@ where } } - pub fn list(&self, index_uuid: Uuid) -> anyhow::Result>> { - let rtxn = self.env.read_txn()?; - let mut updates = Vec::new(); + /// List the updates for `index_uuid`. + pub fn list(&self, index_uuid: Uuid) -> anyhow::Result> { + let mut update_list = BTreeMap::::new(); - let processing = self.processing.read(); - if let Some((uuid, ref processing)) = *processing { + let txn = self.env.read_txn()?; + + let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); + for entry in pendings { + let ((_, uuid, id), pending) = entry?; if uuid == index_uuid { - let update = UpdateStatus::from(processing.clone()); - updates.push(update); + update_list.insert(id, pending.decode()?.into()); } } - let pending = self - .pending_meta - .prefix_iter(&rtxn, index_uuid.as_bytes())? - .filter_map(|entry| { - let (_, p) = entry.ok()?; - if let Some((uuid, ref processing)) = *processing { - // Filter out the currently processing update if it is from this index. - if uuid == index_uuid && processing.id() == p.id() { - None - } else { - Some(UpdateStatus::from(p)) - } - } else { - Some(UpdateStatus::from(p)) - } - }); + let updates = self.updates.prefix_iter(&txn, index_uuid.as_bytes())?; + for entry in updates { + let (_, update) = entry?; + update_list.insert(update.id(), update); + } - updates.extend(pending); + // If the currently processing update is from this index, replace the corresponding pending update with this one. + match *self.state.read() { + State::Processing(uuid, ref processing) if uuid == index_uuid => { + update_list.insert(processing.id(), processing.clone().into()); + } + _ => (), + } - let aborted = self - .aborted_meta - .prefix_iter(&rtxn, index_uuid.as_bytes())? - .filter_map(Result::ok) - .map(|(_, p)| p) - .map(UpdateStatus::from); - - updates.extend(aborted); - - let processed = self - .processed_meta - .iter(&rtxn)? - .filter_map(Result::ok) - .map(|(_, p)| p) - .map(UpdateStatus::from); - - updates.extend(processed); - - let failed = self - .failed_meta - .iter(&rtxn)? - .filter_map(Result::ok) - .map(|(_, p)| p) - .map(UpdateStatus::from); - - updates.extend(failed); - - updates.sort_by_key(|u| u.id()); - - Ok(updates) + Ok(update_list.into_iter().map(|(_, v)| v).collect()) } /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta( - &self, - index_uuid: Uuid, - update_id: u64, - ) -> heed::Result>> { - let rtxn = self.env.read_txn()?; - let key = IndexUuidUpdateId::new(index_uuid, update_id); + pub fn meta(&self, index_uuid: Uuid, update_id: u64) -> heed::Result> { + // Check if the update is the one currently processing + match *self.state.read() { + State::Processing(uuid, ref processing) + if uuid == index_uuid && processing.id() == update_id => + { + return Ok(Some(processing.clone().into())); + } + _ => (), + } - if let Some((uuid, ref meta)) = *self.processing.read() { - if uuid == index_uuid && meta.id() == update_id { - return Ok(Some(UpdateStatus::Processing(meta.clone()))); + let txn = self.env.read_txn()?; + // Else, check if it is in the updates database: + let update = self + .updates + .remap_key_type::() + .get(&txn, &(index_uuid, update_id))?; + + if let Some(update) = update { + return Ok(Some(update)); + } + + // If nothing was found yet, we resolve to iterate over the pending queue. + let pendings = self + .pending_queue + .remap_key_type::() + .iter(&txn)? + .lazily_decode_data(); + + for entry in pendings { + let ((uuid, id), pending) = entry?; + if uuid == index_uuid && id == update_id { + return Ok(Some(pending.decode()?.into())); } } - if let Some(meta) = self - .pending_meta - .remap_key_type::() - .get(&rtxn, &key)? - { - return Ok(Some(UpdateStatus::Enqueued(meta))); - } - - if let Some(meta) = self - .processed_meta - .remap_key_type::() - .get(&rtxn, &key)? - { - return Ok(Some(UpdateStatus::Processed(meta))); - } - - if let Some(meta) = self - .aborted_meta - .remap_key_type::() - .get(&rtxn, &key)? - { - return Ok(Some(UpdateStatus::Aborted(meta))); - } - - if let Some(meta) = self - .failed_meta - .remap_key_type::() - .get(&rtxn, &key)? - { - return Ok(Some(UpdateStatus::Failed(meta))); - } - + // No update was found. Ok(None) } - /// Aborts an update, an aborted update content is deleted and - /// the meta of it is moved into the aborted updates database. - /// - /// Trying to abort an update that is currently being processed, an update - /// that as already been processed or which doesn't actually exist, will - /// return `None`. - #[allow(dead_code)] - pub fn abort_update( - &self, - index_uuid: Uuid, - update_id: u64, - ) -> heed::Result>> { - let mut wtxn = self.env.write_txn()?; - let key = IndexUuidUpdateId::new(index_uuid, update_id); - - // We cannot abort an update that is currently being processed. - if self - .pending_meta - .remap_key_type::() - .first(&wtxn)? - .map(|((_, id), _)| id) - == Some(update_id) - { - return Ok(None); - } - - let pending = match self - .pending_meta - .remap_key_type::() - .get(&wtxn, &key)? - { - Some(meta) => meta, - None => return Ok(None), - }; - - let aborted = pending.abort(); - - self.aborted_meta - .remap_key_type::() - .put(&mut wtxn, &key, &aborted)?; - self.pending_meta - .remap_key_type::() - .delete(&mut wtxn, &key)?; - self.pending - .remap_key_type::() - .delete(&mut wtxn, &key)?; - - wtxn.commit()?; - - Ok(Some(aborted)) - } - - /// Aborts all the pending updates, and not the one being currently processed. - /// Returns the update metas and ids that were successfully aborted. - #[allow(dead_code)] - pub fn abort_pendings(&self, index_uuid: Uuid) -> heed::Result)>> { - let mut wtxn = self.env.write_txn()?; - let mut aborted_updates = Vec::new(); - - // We skip the first pending update as it is currently being processed. - for result in self - .pending_meta - .prefix_iter(&wtxn, index_uuid.as_bytes())? - .remap_key_type::() - .skip(1) - { - let ((_, update_id), pending) = result?; - aborted_updates.push((update_id, pending.abort())); - } - - for (id, aborted) in &aborted_updates { - let key = IndexUuidUpdateId::new(index_uuid, *id); - self.aborted_meta - .remap_key_type::() - .put(&mut wtxn, &key, &aborted)?; - self.pending_meta - .remap_key_type::() - .delete(&mut wtxn, &key)?; - self.pending - .remap_key_type::() - .delete(&mut wtxn, &key)?; - } - - wtxn.commit()?; - - Ok(aborted_updates) - } - - pub fn delete_all(&self, uuid: Uuid) -> anyhow::Result<()> { - fn delete_all( - txn: &mut heed::RwTxn, - uuid: Uuid, - db: Database, - ) -> anyhow::Result<()> - where - A: for<'a> heed::BytesDecode<'a>, - { - let mut iter = db.prefix_iter_mut(txn, uuid.as_bytes())?; - while let Some(_) = iter.next() { - iter.del_current()?; - } - Ok(()) - } - + /// Delete all updates for an index from the update store. + pub fn delete_all(&self, index_uuid: Uuid) -> anyhow::Result<()> { let mut txn = self.env.write_txn()?; + // Contains all the content file paths that we need to be removed if the deletion was successful. + let mut paths_to_remove = Vec::new(); - delete_all(&mut txn, uuid, self.pending)?; - delete_all(&mut txn, uuid, self.pending_meta)?; - delete_all(&mut txn, uuid, self.processed_meta)?; - delete_all(&mut txn, uuid, self.aborted_meta)?; - delete_all(&mut txn, uuid, self.failed_meta)?; + let mut pendings = self.pending_queue.iter_mut(&mut txn)?.lazily_decode_data(); - let processing = self.processing.upgradable_read(); - if let Some((processing_uuid, _)) = *processing { - if processing_uuid == uuid { - parking_lot::RwLockUpgradableReadGuard::upgrade(processing).take(); + while let Some(Ok(((_, uuid, _), pending))) = pendings.next() { + if uuid == index_uuid { + pendings.del_current()?; + let mut pending = pending.decode()?; + if let Some(path) = pending.content.take() { + paths_to_remove.push(path); + } } } + + drop(pendings); + + let mut updates = self + .updates + .prefix_iter_mut(&mut txn, index_uuid.as_bytes())? + .lazily_decode_data(); + + while let Some(_) = updates.next() { + updates.del_current()?; + } + + drop(updates); + + txn.commit()?; + + paths_to_remove.iter().for_each(|path| { + let _ = remove_file(path); + }); + + // We don't care about the currently processing update, since it will be removed by itself + // once its done processing, and we can't abort a running update. + Ok(()) } - pub fn snapshot( - &self, - txn: &mut heed::RwTxn, - path: impl AsRef, - ) -> anyhow::Result<()> { + pub fn snapshot(&self, uuids: &HashSet, path: impl AsRef) -> anyhow::Result<()> { + let state_lock = self.state.write(); + state_lock.swap(State::Snapshoting); + + let txn = self.env.write_txn()?; + let update_path = path.as_ref().join("updates"); create_dir_all(&update_path)?; @@ -571,33 +484,179 @@ where let db_path = update_path.join("data.mdb"); // create db snapshot - self.env - .copy_to_path(&db_path, CompactionOption::Enabled)?; + self.env.copy_to_path(&db_path, CompactionOption::Enabled)?; let update_files_path = update_path.join("update_files"); create_dir_all(&update_files_path)?; - for path in self.pending.iter(&txn)? { - let (_, path) = path?; - let name = path.file_name().unwrap(); - let to = update_files_path.join(name); - copy(path, to)?; + let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); + + for entry in pendings { + let ((_, uuid, _), pending) = entry?; + if uuids.contains(&uuid) { + if let Some(path) = pending.decode()?.content_path() { + let name = path.file_name().unwrap(); + let to = update_files_path.join(name); + copy(path, to)?; + } + } } Ok(()) } - pub fn get_size(&self, txn: &heed::RoTxn) -> anyhow::Result { + pub fn get_info(&self) -> anyhow::Result { let mut size = self.env.size(); + let txn = self.env.read_txn()?; - for path in self.pending.iter(txn)? { - let (_, path) = path?; - - if let Ok(metadata) = path.metadata() { - size += metadata.len() + for entry in self.pending_queue.iter(&txn)? { + let (_, pending) = entry?; + if let Some(path) = pending.content_path() { + size += File::open(path)?.metadata()?.len(); } } - Ok(size) + let processing = match *self.state.read() { + State::Processing(uuid, _) => Some(uuid), + _ => None, + }; + + Ok(UpdateStoreInfo { size, processing }) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::index_controller::{index_actor::MockIndexActorHandle, UpdateResult}; + + use futures::future::ok; + + #[actix_rt::test] + async fn test_next_id() { + let dir = tempfile::tempdir_in(".").unwrap(); + let mut options = EnvOpenOptions::new(); + let handle = Arc::new(MockIndexActorHandle::new()); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir.path(), handle).unwrap(); + + let index1_uuid = Uuid::new_v4(); + let index2_uuid = Uuid::new_v4(); + + let mut txn = update_store.env.write_txn().unwrap(); + let ids = update_store.next_update_id(&mut txn, index1_uuid).unwrap(); + txn.commit().unwrap(); + assert_eq!((0, 0), ids); + + let mut txn = update_store.env.write_txn().unwrap(); + let ids = update_store.next_update_id(&mut txn, index2_uuid).unwrap(); + txn.commit().unwrap(); + assert_eq!((1, 0), ids); + + let mut txn = update_store.env.write_txn().unwrap(); + let ids = update_store.next_update_id(&mut txn, index1_uuid).unwrap(); + txn.commit().unwrap(); + assert_eq!((2, 1), ids); + } + + #[actix_rt::test] + async fn test_register_update() { + let dir = tempfile::tempdir_in(".").unwrap(); + let mut options = EnvOpenOptions::new(); + let handle = Arc::new(MockIndexActorHandle::new()); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir.path(), handle).unwrap(); + let meta = UpdateMeta::ClearDocuments; + let uuid = Uuid::new_v4(); + let store_clone = update_store.clone(); + tokio::task::spawn_blocking(move || { + store_clone + .register_update(meta, Some("here"), uuid) + .unwrap(); + }) + .await + .unwrap(); + + let txn = update_store.env.read_txn().unwrap(); + assert!(update_store + .pending_queue + .get(&txn, &(0, uuid, 0)) + .unwrap() + .is_some()); + } + + #[actix_rt::test] + async fn test_process_update() { + let dir = tempfile::tempdir_in(".").unwrap(); + let mut handle = MockIndexActorHandle::new(); + + handle + .expect_update() + .times(2) + .returning(|_index_uuid, processing, _file| { + if processing.id() == 0 { + Box::pin(ok(Ok(processing.process(UpdateResult::Other)))) + } else { + Box::pin(ok(Err(processing.fail(String::from("err"))))) + } + }); + + let handle = Arc::new(handle); + + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let store = UpdateStore::open(options, dir.path(), handle.clone()).unwrap(); + + // wait a bit for the event loop exit. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + let mut txn = store.env.write_txn().unwrap(); + + let update = Enqueued::new(UpdateMeta::ClearDocuments, 0, None); + let uuid = Uuid::new_v4(); + + store + .pending_queue + .put(&mut txn, &(0, uuid, 0), &update) + .unwrap(); + + let update = Enqueued::new(UpdateMeta::ClearDocuments, 1, None); + + store + .pending_queue + .put(&mut txn, &(1, uuid, 1), &update) + .unwrap(); + + txn.commit().unwrap(); + + // Process the pending, and check that it has been moved to the update databases, and + // removed from the pending database. + let store_clone = store.clone(); + tokio::task::spawn_blocking(move || { + store_clone.process_pending_update(handle.clone()).unwrap(); + store_clone.process_pending_update(handle).unwrap(); + }) + .await + .unwrap(); + + let txn = store.env.read_txn().unwrap(); + + assert!(store.pending_queue.first(&txn).unwrap().is_none()); + let update = store + .updates + .remap_key_type::() + .get(&txn, &(uuid, 0)) + .unwrap() + .unwrap(); + + assert!(matches!(update, UpdateStatus::Processed(_))); + let update = store + .updates + .remap_key_type::() + .get(&txn, &(uuid, 1)) + .unwrap() + .unwrap(); + + assert!(matches!(update, UpdateStatus::Failed(_))); } } diff --git a/meilisearch-http/src/index_controller/update_handler.rs b/meilisearch-http/src/index_controller/update_handler.rs index 1eb622cbf..130b85fe7 100644 --- a/meilisearch-http/src/index_controller/update_handler.rs +++ b/meilisearch-http/src/index_controller/update_handler.rs @@ -6,9 +6,8 @@ use grenad::CompressionType; use milli::update::UpdateBuilder; use rayon::ThreadPool; -use crate::index::UpdateResult; -use crate::index_controller::updates::{Failed, Processed, Processing}; use crate::index_controller::UpdateMeta; +use crate::index_controller::{Failed, Processed, Processing}; use crate::option::IndexerOpts; pub struct UpdateHandler { @@ -59,10 +58,10 @@ impl UpdateHandler { pub fn handle_update( &self, - meta: Processing, - content: File, + meta: Processing, + content: Option, index: Index, - ) -> Result, Failed> { + ) -> Result { use UpdateMeta::*; let update_id = meta.id(); diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 36e327cc2..1515f90e9 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -1,87 +1,121 @@ +use std::path::{Path, PathBuf}; + use chrono::{DateTime, Utc}; +use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct Enqueued { - pub update_id: u64, - pub meta: M, - pub enqueued_at: DateTime, +use crate::index::{Facets, Settings}; + +pub type UpdateError = String; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum UpdateResult { + DocumentsAddition(DocumentAdditionResult), + DocumentDeletion { deleted: u64 }, + Other, } -impl Enqueued { - pub fn new(meta: M, update_id: u64) -> Self { +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum UpdateMeta { + DocumentsAddition { + method: IndexDocumentsMethod, + format: UpdateFormat, + primary_key: Option, + }, + ClearDocuments, + DeleteDocuments, + Settings(Settings), + Facets(Facets), +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Enqueued { + pub update_id: u64, + pub meta: UpdateMeta, + pub enqueued_at: DateTime, + pub content: Option, +} + +impl Enqueued { + pub fn new(meta: UpdateMeta, update_id: u64, content: Option) -> Self { Self { enqueued_at: Utc::now(), meta, update_id, + content, } } - pub fn processing(self) -> Processing { + pub fn processing(self) -> Processing { Processing { from: self, started_processing_at: Utc::now(), } } - pub fn abort(self) -> Aborted { + pub fn abort(self) -> Aborted { Aborted { from: self, aborted_at: Utc::now(), } } - pub fn meta(&self) -> &M { + pub fn meta(&self) -> &UpdateMeta { &self.meta } pub fn id(&self) -> u64 { self.update_id } + + pub fn content_path(&self) -> Option<&Path> { + self.content.as_deref() + } } -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] -pub struct Processed { - pub success: N, +pub struct Processed { + pub success: UpdateResult, pub processed_at: DateTime, #[serde(flatten)] - pub from: Processing, + pub from: Processing, } -impl Processed { +impl Processed { pub fn id(&self) -> u64 { self.from.id() } } -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] -pub struct Processing { +pub struct Processing { #[serde(flatten)] - pub from: Enqueued, + pub from: Enqueued, pub started_processing_at: DateTime, } -impl Processing { +impl Processing { pub fn id(&self) -> u64 { self.from.id() } - pub fn meta(&self) -> &M { + pub fn meta(&self) -> &UpdateMeta { self.from.meta() } - pub fn process(self, meta: N) -> Processed { + pub fn process(self, success: UpdateResult) -> Processed { Processed { - success: meta, + success, from: self, processed_at: Utc::now(), } } - pub fn fail(self, error: E) -> Failed { + pub fn fail(self, error: UpdateError) -> Failed { Failed { from: self, error, @@ -90,46 +124,46 @@ impl Processing { } } -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] -pub struct Aborted { +pub struct Aborted { #[serde(flatten)] - from: Enqueued, + from: Enqueued, aborted_at: DateTime, } -impl Aborted { +impl Aborted { pub fn id(&self) -> u64 { self.from.id() } } -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] -pub struct Failed { +pub struct Failed { #[serde(flatten)] - from: Processing, - error: E, + from: Processing, + error: UpdateError, failed_at: DateTime, } -impl Failed { +impl Failed { pub fn id(&self) -> u64 { self.from.id() } } -#[derive(Debug, PartialEq, Eq, Hash, Serialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(tag = "status", rename_all = "camelCase")] -pub enum UpdateStatus { - Processing(Processing), - Enqueued(Enqueued), - Processed(Processed), - Aborted(Aborted), - Failed(Failed), +pub enum UpdateStatus { + Processing(Processing), + Enqueued(Enqueued), + Processed(Processed), + Aborted(Aborted), + Failed(Failed), } -impl UpdateStatus { +impl UpdateStatus { pub fn id(&self) -> u64 { match self { UpdateStatus::Processing(u) => u.id(), @@ -140,7 +174,7 @@ impl UpdateStatus { } } - pub fn processed(&self) -> Option<&Processed> { + pub fn processed(&self) -> Option<&Processed> { match self { UpdateStatus::Processed(p) => Some(p), _ => None, @@ -148,32 +182,32 @@ impl UpdateStatus { } } -impl From> for UpdateStatus { - fn from(other: Enqueued) -> Self { +impl From for UpdateStatus { + fn from(other: Enqueued) -> Self { Self::Enqueued(other) } } -impl From> for UpdateStatus { - fn from(other: Aborted) -> Self { +impl From for UpdateStatus { + fn from(other: Aborted) -> Self { Self::Aborted(other) } } -impl From> for UpdateStatus { - fn from(other: Processed) -> Self { +impl From for UpdateStatus { + fn from(other: Processed) -> Self { Self::Processed(other) } } -impl From> for UpdateStatus { - fn from(other: Processing) -> Self { +impl From for UpdateStatus { + fn from(other: Processing) -> Self { Self::Processing(other) } } -impl From> for UpdateStatus { - fn from(other: Failed) -> Self { +impl From for UpdateStatus { + fn from(other: Failed) -> Self { Self::Failed(other) } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 27ffaa05e..253326276 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{collections::HashSet, path::PathBuf}; use log::{info, warn}; use tokio::sync::mpsc; @@ -78,7 +78,7 @@ impl UuidResolverActor { Ok(result) } - async fn handle_snapshot(&self, path: PathBuf) -> Result> { + async fn handle_snapshot(&self, path: PathBuf) -> Result> { self.store.snapshot(path).await } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index c522e87e6..db4c482bd 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; @@ -67,7 +68,7 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - async fn snapshot(&self, path: PathBuf) -> Result> { + async fn snapshot(&self, path: PathBuf) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::SnapshotRequest { path, ret }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index e7d29f05f..a72bf0587 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::path::PathBuf; use tokio::sync::oneshot; @@ -28,7 +29,7 @@ pub enum UuidResolveMsg { }, SnapshotRequest { path: PathBuf, - ret: oneshot::Sender>>, + ret: oneshot::Sender>>, }, GetSize { ret: oneshot::Sender>, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 33a089ddb..ef17133ff 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -3,6 +3,7 @@ mod handle_impl; mod message; mod store; +use std::collections::HashSet; use std::path::PathBuf; use thiserror::Error; @@ -29,7 +30,7 @@ pub trait UuidResolverHandle { async fn create(&self, name: String) -> anyhow::Result; async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; - async fn snapshot(&self, path: PathBuf) -> Result>; + async fn snapshot(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 1f057830b..29c034c44 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -1,5 +1,6 @@ -use std::fs::create_dir_all; use std::path::{Path, PathBuf}; +use std::collections::HashSet; +use std::fs::create_dir_all; use heed::{ types::{ByteSlice, Str}, @@ -19,7 +20,7 @@ pub trait UuidStore { async fn delete(&self, uid: String) -> Result>; async fn list(&self) -> Result>; async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; - async fn snapshot(&self, path: PathBuf) -> Result>; + async fn snapshot(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } @@ -129,17 +130,17 @@ impl UuidStore for HeedUuidStore { .await? } - async fn snapshot(&self, mut path: PathBuf) -> Result> { + async fn snapshot(&self, mut path: PathBuf) -> Result> { let env = self.env.clone(); let db = self.db; tokio::task::spawn_blocking(move || { // Write transaction to acquire a lock on the database. let txn = env.write_txn()?; - let mut entries = Vec::new(); + let mut entries = HashSet::new(); for entry in db.iter(&txn)? { let (_, uuid) = entry?; let uuid = Uuid::from_slice(uuid)?; - entries.push(uuid) + entries.insert(uuid); } // only perform snapshot if there are indexes diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 357a2b16a..4f211bf09 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -107,14 +107,11 @@ async fn get_all_documents( path: web::Path, params: web::Query, ) -> Result { - let attributes_to_retrieve = params - .attributes_to_retrieve - .as_ref() - .and_then(|attrs| { + let attributes_to_retrieve = params.attributes_to_retrieve.as_ref().and_then(|attrs| { let mut names = Vec::new(); for name in attrs.split(',').map(String::from) { if name == "*" { - return None + return None; } names.push(name); } diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 86aa80c3d..adb7fef3e 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -185,12 +185,9 @@ impl Index<'_> { self.service.get(url).await } - make_settings_test_routes!( - distinct_attribute - ); + make_settings_test_routes!(distinct_attribute); } - pub struct GetDocumentOptions; #[derive(Debug, Default)] diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 87e5cecb7..9de5fe9db 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -77,8 +77,8 @@ async fn document_addition_with_primary_key() { "content": "foo", } ]); - let (_response, code) = index.add_documents(documents, Some("primary")).await; - assert_eq!(code, 202); + let (response, code) = index.add_documents(documents, Some("primary")).await; + assert_eq!(code, 202, "response: {}", response); index.wait_update_id(0).await; @@ -189,8 +189,8 @@ async fn replace_document() { } ]); - let (_response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 202); + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202, "response: {}", response); index.wait_update_id(0).await; @@ -260,8 +260,8 @@ async fn update_document() { } ]); - let (_response, code) = index.update_documents(documents, None).await; - assert_eq!(code, 202); + let (response, code) = index.update_documents(documents, None).await; + assert_eq!(code, 202, "response: {}", response); index.wait_update_id(1).await; diff --git a/meilisearch-http/tests/settings/distinct.rs b/meilisearch-http/tests/settings/distinct.rs index a3aec6baf..818f200fd 100644 --- a/meilisearch-http/tests/settings/distinct.rs +++ b/meilisearch-http/tests/settings/distinct.rs @@ -6,14 +6,18 @@ async fn set_and_reset_distinct_attribute() { let server = Server::new().await; let index = server.index("test"); - let (_response, _code) = index.update_settings(json!({ "distinctAttribute": "test"})).await; + let (_response, _code) = index + .update_settings(json!({ "distinctAttribute": "test"})) + .await; index.wait_update_id(0).await; let (response, _) = index.settings().await; assert_eq!(response["distinctAttribute"], "test"); - index.update_settings(json!({ "distinctAttribute": null })).await; + index + .update_settings(json!({ "distinctAttribute": null })) + .await; index.wait_update_id(1).await; diff --git a/meilisearch-http/tests/settings/mod.rs b/meilisearch-http/tests/settings/mod.rs index b7102cc5f..05339cb37 100644 --- a/meilisearch-http/tests/settings/mod.rs +++ b/meilisearch-http/tests/settings/mod.rs @@ -1,2 +1,2 @@ -mod get_settings; mod distinct; +mod get_settings; From bb79a15c042793cc48fb81a6e3de2299c64b5479 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 27 Apr 2021 15:29:00 +0200 Subject: [PATCH 295/522] reenable ranking rules route --- meilisearch-http/src/routes/settings/mod.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 54cc53d7e..c4403d9d7 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -103,11 +103,11 @@ make_setting_route!( distinct_attribute ); -//make_setting_route!( -//"/indexes/{index_uid}/settings/ranking-rules", -//Vec, -//ranking_rules -//); +make_setting_route!( + "/indexes/{index_uid}/settings/ranking-rules", + Vec, + ranking_rules +); macro_rules! create_services { ($($mod:ident),*) => { @@ -130,7 +130,8 @@ create_services!( displayed_attributes, searchable_attributes, distinct_attribute, - stop_words + stop_words, + ranking_rules ); #[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] From 703d2026e4ea16c582b43ce03e50c0d5f76c53a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 27 Apr 2021 16:33:00 +0200 Subject: [PATCH 296/522] Update README.md --- README.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0271ef082..6b80fad0e 100644 --- a/README.md +++ b/README.md @@ -1 +1,35 @@ -# transplant of meilisearch http on the new engine +# Transplant + +Transplant makes communication between the users and [Milli](https://github.com/meilisearch/milli) using HTTP. The final purpose of Transplant is to be merged into the current [MeiliSearch repository](https://github.com/meilisearch/MeiliSearch) so that users will enjoy the new search engine performance provided by Milli. + +## Run the alpha releases + +Currently only alpha versions are available. + +You can: + +- Run it with Docker, for instance: + +```bash +docker run -p 7700:7700 getmeili/meilisearch:v0.21.0-alpha.4 ./meilisearch +``` + +- With the available [release assets](https://github.com/meilisearch/transplant/releases). + +- Compile from the source code: + +```bash +cargo run --release +``` + +## Run the tests + +``` +cargo test +``` + +If you encouter any `Too many open files` error when running the tests, please upgrade the maximum number of open file descriptors with this command: + +``` +ulimit -Sn 3000 +``` From cea0c1f41d56138519ba730bb624e01cd51c04d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 27 Apr 2021 16:33:22 +0200 Subject: [PATCH 297/522] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b80fad0e..7e0fc476f 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ cargo run --release cargo test ``` -If you encouter any `Too many open files` error when running the tests, please upgrade the maximum number of open file descriptors with this command: +If you encounter any `Too many open files` error when running the tests, please upgrade the maximum number of open file descriptors with this command: ``` ulimit -Sn 3000 From a961f0ce75c7035d40558ca151eae14398d81f51 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 27 Apr 2021 17:51:12 +0200 Subject: [PATCH 298/522] fix clippy warnings --- Cargo.lock | 2 ++ meilisearch-http/src/data/mod.rs | 6 +----- .../src/index_controller/index_actor/actor.rs | 6 +----- .../src/index_controller/index_actor/mod.rs | 6 +----- .../src/index_controller/update_actor/actor.rs | 7 +------ .../index_controller/update_actor/update_store.rs | 14 ++++---------- 6 files changed, 10 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cd5e26a6..f802ac4ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" version = "0.4.0" diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index e63bf9380..ec64192f7 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -68,11 +68,7 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { - index_controller, - options, - api_keys, - }; + let inner = DataInner { index_controller, api_keys, options }; let inner = Arc::new(inner); Ok(Data { inner }) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 05130e795..06d2b0f60 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -33,11 +33,7 @@ impl IndexActor { let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; let update_handler = Arc::new(update_handler); let receiver = Some(receiver); - Ok(Self { - receiver, - store, - update_handler, - }) + Ok(Self { receiver, update_handler, store }) } pub async fn run(mut self) { diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index d49923fa0..ef5ea524c 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -44,11 +44,7 @@ impl IndexMeta { let created_at = index.created_at(&txn)?; let updated_at = index.updated_at(&txn)?; let primary_key = index.primary_key(&txn)?.map(String::from); - Ok(Self { - primary_key, - updated_at, - created_at, - }) + Ok(Self { created_at, updated_at, primary_key }) } } diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index c98df7d25..e47edc5bc 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -44,12 +44,7 @@ where let store = UpdateStore::open(options, &path, index_handle.clone())?; std::fs::create_dir_all(path.join("update_files"))?; assert!(path.exists()); - Ok(Self { - store, - inbox, - path, - index_handle, - }) + Ok(Self { path, store, inbox, index_handle }) } pub async fn run(mut self) { diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 795b6c36a..6a916af33 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -58,7 +58,7 @@ impl StateLock { fn from_state(state: State) -> Self { let lock = Mutex::new(()); let data = ArcSwap::from(Arc::new(state)); - Self { data, lock } + Self { lock, data } } fn read(&self) -> Arc { @@ -68,10 +68,11 @@ impl StateLock { fn write(&self) -> StateLockGuard { let _lock = self.lock.lock(); let state = &self; - StateLockGuard { state, _lock } + StateLockGuard { _lock, state } } } +#[allow(clippy::large_enum_variant)] pub enum State { Idle, Processing(Uuid, Processing), @@ -201,14 +202,7 @@ impl UpdateStore { .try_send(()) .expect("Failed to init update store"); - let update_store = Arc::new(UpdateStore { - env, - notification_sender, - next_update_id, - pending_queue, - updates, - state, - }); + let update_store = Arc::new(UpdateStore { env, pending_queue, next_update_id, updates, state, notification_sender }); // We need a weak reference so we can take ownership on the arc later when we // want to close the index. From 3ee2b079186b2443006a0ccae34df35758a8a41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 29 Apr 2021 15:19:48 +0200 Subject: [PATCH 299/522] Improve CI --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bdbf50a95..ee23eb487 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -24,8 +24,8 @@ jobs: - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: - command: check - args: --no-default-features + command: build + args: --locked --release --no-default-features - name: Run cargo test uses: actions-rs/cargo@v1 with: From 3d5fba94c234c1ce5b3f0203710f45be850cbf77 Mon Sep 17 00:00:00 2001 From: Morgane Dubus Date: Thu, 29 Apr 2021 15:22:41 +0200 Subject: [PATCH 300/522] Update mini-dashboard with version 0.1.2 --- meilisearch-http/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e6912e428..a2f254834 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -117,5 +117,5 @@ default = ["telemetry", "mini-dashboard"] jemallocator = "0.3.2" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.1/build.zip" -sha1 = "f4247f8f534214e2811637e0555347c3f6bf5794" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.2/build.zip" +sha1 = "881949457a710a22b5cefcb0418038ceb8b0eb26" From 78217bcf1842bab67177070b1ac652e4cff594d2 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 29 Apr 2021 16:19:06 +0200 Subject: [PATCH 301/522] Fix cors authentication issue --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 1 + .../src/helpers/authentication.rs | 109 +++++++++++++----- 3 files changed, 79 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f802ac4ab..62cd4b300 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1771,6 +1771,7 @@ dependencies = [ "oxidized-json-checker", "parking_lot", "paste", + "pin-project", "rand 0.7.3", "rayon", "regex", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e6912e428..e32ebf587 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -73,6 +73,7 @@ tokio = { version = "1", features = ["full"] } uuid = { version = "0.8.2", features = ["serde"] } walkdir = "2.3.2" obkv = "0.1.1" +pin-project = "1.0.7" [dependencies.sentry] default-features = false diff --git a/meilisearch-http/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs index 9944c0bd4..a1a0c431e 100644 --- a/meilisearch-http/src/helpers/authentication.rs +++ b/meilisearch-http/src/helpers/authentication.rs @@ -1,14 +1,16 @@ -use std::cell::RefCell; use std::pin::Pin; -use std::rc::Rc; use std::task::{Context, Poll}; use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; use actix_web::web; -use futures::future::{err, ok, Future, Ready}; +use actix_web::body::Body; +use futures::ready; +use futures::future::{ok, Future, Ready}; +use actix_web::ResponseError as _; +use pin_project::pin_project; -use crate::error::{Error, ResponseError}; use crate::Data; +use crate::error::{Error, ResponseError}; #[derive(Clone, Copy)] pub enum Authentication { @@ -17,13 +19,11 @@ pub enum Authentication { Admin, } -impl Transform for Authentication +impl Transform for Authentication where - S: Service, Error = actix_web::Error>, - S::Future: 'static, - B: 'static, + S: Service, Error = actix_web::Error>, { - type Response = ServiceResponse; + type Response = ServiceResponse; type Error = actix_web::Error; type InitError = (); type Transform = LoggingMiddleware; @@ -32,54 +32,45 @@ where fn new_transform(&self, service: S) -> Self::Future { ok(LoggingMiddleware { acl: *self, - service: Rc::new(RefCell::new(service)), + service, }) } } pub struct LoggingMiddleware { acl: Authentication, - service: Rc>, + service: S, } #[allow(clippy::type_complexity)] -impl Service for LoggingMiddleware +impl Service for LoggingMiddleware where - S: Service, Error = actix_web::Error> + 'static, - S::Future: 'static, - B: 'static, + S: Service, Error = actix_web::Error>, { - type Response = ServiceResponse; + type Response = ServiceResponse; type Error = actix_web::Error; - type Future = Pin>>>; + type Future = AuthenticationFuture; fn poll_ready(&self, cx: &mut Context) -> Poll> { self.service.poll_ready(cx) } fn call(&self, req: ServiceRequest) -> Self::Future { - let svc = self.service.clone(); - // This unwrap is left because this error should never appear. If that's the case, then - // it means that actix-web has an issue or someone changes the type `Data`. let data = req.app_data::>().unwrap(); if data.api_keys().master.is_none() { - return Box::pin(svc.call(req)); + return AuthenticationFuture::Authenticated(self.service.call(req)) } let auth_header = match req.headers().get("X-Meili-API-Key") { Some(auth) => match auth.to_str() { Ok(auth) => auth, Err(_) => { - return Box::pin(err( - ResponseError::from(Error::MissingAuthorizationHeader).into() - )) + return AuthenticationFuture::NoHeader(Some(req)) } }, None => { - return Box::pin(err( - ResponseError::from(Error::MissingAuthorizationHeader).into() - )); + return AuthenticationFuture::NoHeader(Some(req)) } }; @@ -97,12 +88,66 @@ where }; if authenticated { - Box::pin(svc.call(req)) + AuthenticationFuture::Authenticated(self.service.call(req)) } else { - Box::pin(err(ResponseError::from(Error::InvalidToken( - auth_header.to_string(), - )) - .into())) + AuthenticationFuture::Refused(Some(req)) + } + } +} + +#[pin_project(project = AuthProj)] +pub enum AuthenticationFuture +where + S: Service, +{ + Authenticated(#[pin] S::Future), + NoHeader(Option), + Refused(Option), +} + +impl Future for AuthenticationFuture +where + S: Service, Error = actix_web::Error>, +{ + type Output = Result, actix_web::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) ->Poll { + let this = self.project(); + match this { + AuthProj::Authenticated(fut) => { + match ready!(fut.poll(cx)) { + Ok(resp) => Poll::Ready(Ok(resp)), + Err(e) => Poll::Ready(Err(e)), + } + } + AuthProj::NoHeader(req) => { + match req.take() { + Some(req) => { + let response = ResponseError::from(Error::MissingAuthorizationHeader); + let response = response.error_response(); + let response = req.into_response(response); + Poll::Ready(Ok(response)) + } + // https://doc.rust-lang.org/nightly/std/future/trait.Future.html#panics + None => unreachable!("poll called again on ready future"), + } + } + AuthProj::Refused(req) => { + match req.take() { + Some(req) => { + let bad_token = req.headers() + .get("X-Meili-API-Key") + .map(|h| h.to_str().map(String::from).unwrap_or_default()) + .unwrap_or_default(); + let response = ResponseError::from(Error::InvalidToken(bad_token)); + let response = response.error_response(); + let response = req.into_response(response); + Poll::Ready(Ok(response)) + } + // https://doc.rust-lang.org/nightly/std/future/trait.Future.html#panics + None => unreachable!("poll called again on ready future"), + } + } } } } From 928fb34effc560d6f6f96de166e9a9d3e4ad57a9 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 4 May 2021 15:10:22 +0200 Subject: [PATCH 302/522] bump milli and fix tests --- Cargo.lock | 25 +++++++++++++++---- meilisearch-http/Cargo.toml | 2 +- .../tests/documents/add_documents.rs | 9 +++---- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62cd4b300..778652bca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1761,7 +1761,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.1", "memmap", "milli", "mime", @@ -1814,6 +1814,22 @@ dependencies = [ "whatlang", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.2.2" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.2#eda4ed4968c8ac973cf1707ef89bd7012bb2722f" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "memchr" version = "2.3.4" @@ -1841,8 +1857,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.1.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.1.1#f5ec14c54cc3ab76ca581eca9b6e0870f09fd63f" +version = "0.2.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.2.0#792225eaffce6b3682f9b30b7370b6a547c4757e" dependencies = [ "anyhow", "bstr", @@ -1862,9 +1878,8 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.2", "memmap", - "num-traits", "obkv", "once_cell", "ordered-float", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 12ae6e3b1..68f76ebf9 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.0" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 9de5fe9db..1ec84e046 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -140,9 +140,7 @@ async fn add_documents_with_primary_key_and_primary_key_already_exists() { let (response, code) = index.get_update(0).await; assert_eq!(code, 200); - assert_eq!(response["status"], "processed"); - assert_eq!(response["updateId"], 0); - assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + assert_eq!(response["status"], "failed"); let (response, code) = index.get().await; assert_eq!(code, 200); @@ -168,9 +166,8 @@ async fn update_documents_with_primary_key_and_primary_key_already_exists() { index.wait_update_id(0).await; let (response, code) = index.get_update(0).await; assert_eq!(code, 200); - assert_eq!(response["status"], "processed"); - assert_eq!(response["updateId"], 0); - assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + // Documents without a primary key are not accepted. + assert_eq!(response["status"], "failed"); let (response, code) = index.get().await; assert_eq!(code, 200); From eb03a3ccb16ccba703e9847f9d292a599ff93d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 3 May 2021 15:39:53 +0200 Subject: [PATCH 303/522] Upgrade Milli and Tokenizer --- Cargo.lock | 185 ++++++++++++++++-------------------- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 85 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 778652bca..f1c109a79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "actix-codec" version = "0.4.0" @@ -54,7 +52,7 @@ dependencies = [ "flate2", "futures-core", "futures-util", - "h2 0.3.2", + "h2 0.3.3", "http", "httparse", "itoa", @@ -83,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -215,7 +213,7 @@ checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -266,9 +264,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -321,7 +319,7 @@ checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -332,7 +330,7 @@ checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -354,11 +352,12 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.57" +version = "0.3.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ed203b9ba68b242c62b3fb7480f589dd49829be1edb3fe8fc8b4ffda2dcb8d" +checksum = "88fb5a785d6b44fd9d6700935608639af1b8356de1e55d5f7c2740f4faa15d82" dependencies = [ "addr2line", + "cc", "cfg-if 1.0.0", "libc", "miniz_oxide", @@ -460,9 +459,9 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" +checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279" dependencies = [ "lazy_static", "memchr", @@ -691,7 +690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", + "crossbeam-utils 0.8.4", ] [[package]] @@ -702,17 +701,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.3", + "crossbeam-utils 0.8.4", ] [[package]] name = "crossbeam-epoch" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" +checksum = "52fb27eab85b17fbb9f6fd667089e07d6a2eb8743d02639ee7f6a7a7729c9c94" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", + "crossbeam-utils 0.8.4", "lazy_static", "memoffset", "scopeguard", @@ -739,9 +738,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -789,7 +788,7 @@ dependencies = [ "convert_case", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -893,7 +892,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", "synstructure", ] @@ -1049,7 +1048,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -1182,9 +1181,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" dependencies = [ "bytes 1.0.1", "fnv", @@ -1391,15 +1390,15 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f006b8784cfb01fe7aa9c46f5f5cd4cf5c85a8c612a0653ec97642979062665" +checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" dependencies = [ "bytes 1.0.1", "futures-channel", "futures-core", "futures-util", - "h2 0.3.2", + "h2 0.3.3", "http", "http-body 0.4.1", "httparse", @@ -1436,7 +1435,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" dependencies = [ "futures-util", - "hyper 0.14.6", + "hyper 0.14.7", "log", "rustls 0.19.1", "tokio 1.5.0", @@ -1615,9 +1614,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" [[package]] name = "linked-hash-map" @@ -1656,9 +1655,9 @@ checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" [[package]] name = "lock_api" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] @@ -1761,7 +1760,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer 0.2.1", + "meilisearch-tokenizer", "memmap", "milli", "mime", @@ -1798,22 +1797,6 @@ dependencies = [ "zip", ] -[[package]] -name = "meilisearch-tokenizer" -version = "0.2.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - [[package]] name = "meilisearch-tokenizer" version = "0.2.2" @@ -1832,9 +1815,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memmap" @@ -1878,7 +1861,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer 0.2.2", + "meilisearch-tokenizer", "memmap", "obkv", "once_cell", @@ -2001,7 +1984,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -2103,9 +2086,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "ordered-float" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" +checksum = "b50b8919aecb97e5ee9aceef27e24f39c46b11831130f4a6b7b091ec5de0de12" dependencies = [ "num-traits", ] @@ -2215,7 +2198,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -2284,7 +2267,7 @@ checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -2319,9 +2302,9 @@ checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" [[package]] name = "predicates" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" +checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" dependencies = [ "difference", "float-cmp", @@ -2355,7 +2338,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", "version_check", ] @@ -2397,7 +2380,7 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" dependencies = [ - "unicode-xid 0.2.1", + "unicode-xid 0.2.2", ] [[package]] @@ -2572,7 +2555,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.3", + "crossbeam-utils 0.8.4", "lazy_static", "num_cpus", ] @@ -2588,18 +2571,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.4.5" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b" dependencies = [ "aho-corasick", "memchr", @@ -2617,9 +2600,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.23" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "remove_dir_all" @@ -2680,7 +2663,7 @@ dependencies = [ "futures-util", "http", "http-body 0.4.1", - "hyper 0.14.6", + "hyper 0.14.7", "hyper-rustls 0.22.1", "ipnet", "js-sys", @@ -2725,9 +2708,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4b2e7ab0bbb2d144558ae3f4761a0db06d21463b45756fc64c3393cdba3d447" +checksum = "536cfa885fc388b8ae69edf96d7970849b7d9c1395da1b8330f17715babf8a09" dependencies = [ "bytemuck", "byteorder", @@ -2736,9 +2719,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" +checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce" [[package]] name = "rustc_version" @@ -2904,7 +2887,7 @@ checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3040,9 +3023,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a" +checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" [[package]] name = "socket2" @@ -3104,7 +3087,7 @@ dependencies = [ "quote 1.0.9", "serde", "serde_derive", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3120,7 +3103,7 @@ dependencies = [ "serde_derive", "serde_json", "sha1", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3156,7 +3139,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3172,13 +3155,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.70" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9505f307c872bab8eb46f77ae357c8eba1fdacead58ee5a850116b1d7f82883" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "unicode-xid 0.2.1", + "unicode-xid 0.2.2", ] [[package]] @@ -3198,8 +3181,8 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", - "unicode-xid 0.2.1", + "syn 1.0.72", + "unicode-xid 0.2.2", ] [[package]] @@ -3272,7 +3255,7 @@ checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3321,7 +3304,7 @@ dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", "standback", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3395,7 +3378,7 @@ checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3466,9 +3449,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", "log", @@ -3478,9 +3461,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" dependencies = [ "lazy_static", ] @@ -3575,9 +3558,9 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "untrusted" @@ -3606,9 +3589,9 @@ checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" [[package]] name = "utf8-width" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" +checksum = "7cf7d77f457ef8dfa11e4cd5933c5ddb5dc52a94664071951219a97710f0a32b" [[package]] name = "uuid" @@ -3699,7 +3682,7 @@ dependencies = [ "log", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", "wasm-bindgen-shared", ] @@ -3733,7 +3716,7 @@ checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3874,12 +3857,12 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc9c39e6d503229ffa00cc2954af4a751e6bbedf2a2c18e856eb3ece93d32495" +checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ "proc-macro2 1.0.26", - "syn 1.0.70", + "syn 1.0.72", "synstructure", ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 68f76ebf9..7ac3ecb38 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -49,7 +49,7 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.0" } mime = "0.3.16" From a717925caa62d5293e7d476bc6ce4060a858a995 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 4 May 2021 18:20:56 +0200 Subject: [PATCH 304/522] remove filters, rename facet_filters to filter --- meilisearch-http/src/index/search.rs | 7 +++---- meilisearch-http/src/routes/search.rs | 8 +++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 0ff6c1bc3..312067472 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -33,9 +33,8 @@ pub struct SearchQuery { pub attributes_to_crop: Option>, pub crop_length: Option, pub attributes_to_highlight: Option>, - pub filters: Option, pub matches: Option, - pub facet_filters: Option, + pub filter: Option, pub facet_distributions: Option>, } @@ -75,8 +74,8 @@ impl Index { search.limit(query.limit); search.offset(query.offset.unwrap_or_default()); - if let Some(ref facets) = query.facet_filters { - if let Some(facets) = parse_facets(facets, self, &rtxn)? { + if let Some(ref filter) = query.filter { + if let Some(facets) = parse_facets(filter, self, &rtxn)? { search.facet_condition(facets); } } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 86beb2750..5f4e285f8 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -24,9 +24,8 @@ pub struct SearchQueryGet { attributes_to_crop: Option, crop_length: Option, attributes_to_highlight: Option, - filters: Option, + filter: Option, matches: Option, - facet_filters: Option, facet_distributions: Option, } @@ -50,7 +49,7 @@ impl TryFrom for SearchQuery { .facet_distributions .map(|attrs| attrs.split(',').map(String::from).collect::>()); - let facet_filters = match other.facet_filters { + let filter = match other.filter { Some(ref f) => Some(serde_json::from_str(f)?), None => None, }; @@ -63,9 +62,8 @@ impl TryFrom for SearchQuery { attributes_to_crop, crop_length: other.crop_length, attributes_to_highlight, - filters: other.filters, + filter, matches: other.matches, - facet_filters, facet_distributions, }) } From ec7eb7798f209c855cdc867994fb14d2dd2d2768 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 4 May 2021 22:36:31 +0200 Subject: [PATCH 305/522] remove facet setting --- meilisearch-http/src/index/updates.rs | 23 ------------------- .../src/index_controller/update_handler.rs | 1 - .../src/index_controller/updates.rs | 3 +-- 3 files changed, 1 insertion(+), 26 deletions(-) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 038f1f7e6..4afd2e297 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -196,29 +196,6 @@ impl Index { } } - pub fn update_facets( - &self, - levels: &Facets, - update_builder: UpdateBuilder, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.write_txn()?; - let mut builder = update_builder.facets(&mut wtxn, self); - if let Some(value) = levels.level_group_size { - builder.level_group_size(value); - } - if let Some(value) = levels.min_level_size { - builder.min_level_size(value); - } - match builder.execute() { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e), - } - } - pub fn delete_documents( &self, document_ids: Option, diff --git a/meilisearch-http/src/index_controller/update_handler.rs b/meilisearch-http/src/index_controller/update_handler.rs index 130b85fe7..d0086aadd 100644 --- a/meilisearch-http/src/index_controller/update_handler.rs +++ b/meilisearch-http/src/index_controller/update_handler.rs @@ -83,7 +83,6 @@ impl UpdateHandler { ClearDocuments => index.clear_documents(update_builder), DeleteDocuments => index.delete_documents(content, update_builder), Settings(settings) => index.update_settings(settings, update_builder), - Facets(levels) => index.update_facets(levels, update_builder), }; match result { diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 1515f90e9..02827abd0 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; -use crate::index::{Facets, Settings}; +use crate::index::Settings; pub type UpdateError = String; @@ -26,7 +26,6 @@ pub enum UpdateMeta { ClearDocuments, DeleteDocuments, Settings(Settings), - Facets(Facets), } #[derive(Debug, Serialize, Deserialize, Clone)] From b192cb9c1f243bee805af50c6ce24d33260ff732 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 4 May 2021 18:22:48 +0200 Subject: [PATCH 306/522] enable string syntax for the filters --- meilisearch-http/src/index/search.rs | 61 +++++++++++++-------------- meilisearch-http/src/routes/search.rs | 8 +++- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 312067472..6f5a0d6d9 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -277,35 +277,6 @@ impl Matcher for MatchingWords { } } -fn parse_facets_array( - txn: &RoTxn, - index: &Index, - arr: &[Value], -) -> anyhow::Result> { - let mut ands = Vec::new(); - for value in arr { - match value { - Value::String(s) => ands.push(Either::Right(s.clone())), - Value::Array(arr) => { - let mut ors = Vec::new(); - for value in arr { - match value { - Value::String(s) => ors.push(s.clone()), - v => bail!("Invalid facet expression, expected String, found: {:?}", v), - } - } - ands.push(Either::Left(ors)); - } - v => bail!( - "Invalid facet expression, expected String or [String], found: {:?}", - v - ), - } - } - - FacetCondition::from_array(txn, &index.0, ands) -} - struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, marks: (String, String), @@ -367,13 +338,41 @@ fn parse_facets( txn: &RoTxn, ) -> anyhow::Result> { match facets { - // Disabled for now - //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), + Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), Value::Array(arr) => parse_facets_array(txn, index, arr), v => bail!("Invalid facet expression, expected Array, found: {:?}", v), } } +fn parse_facets_array( + txn: &RoTxn, + index: &Index, + arr: &[Value], +) -> anyhow::Result> { + let mut ands = Vec::new(); + for value in arr { + match value { + Value::String(s) => ands.push(Either::Right(s.clone())), + Value::Array(arr) => { + let mut ors = Vec::new(); + for value in arr { + match value { + Value::String(s) => ors.push(s.clone()), + v => bail!("Invalid facet expression, expected String, found: {:?}", v), + } + } + ands.push(Either::Left(ors)); + } + v => bail!( + "Invalid facet expression, expected String or [String], found: {:?}", + v + ), + } + } + + FacetCondition::from_array(txn, &index.0, ands) +} + #[cfg(test)] mod test { use std::iter::FromIterator; diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 5f4e285f8..be06960cf 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use std::convert::{TryFrom, TryInto}; use actix_web::{get, post, web, HttpResponse}; +use serde_json::Value; use serde::Deserialize; use crate::error::ResponseError; @@ -50,7 +51,12 @@ impl TryFrom for SearchQuery { .map(|attrs| attrs.split(',').map(String::from).collect::>()); let filter = match other.filter { - Some(ref f) => Some(serde_json::from_str(f)?), + Some(f) => { + match serde_json::from_str(&f) { + Ok(v) => Some(v), + _ => Some(Value::String(f)), + } + }, None => None, }; From 706643dfedf3f53182d9c424433b6db875258022 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 10 May 2021 17:30:09 +0200 Subject: [PATCH 307/522] type setting struct --- meilisearch-http/src/data/mod.rs | 4 +-- meilisearch-http/src/data/updates.rs | 4 +-- meilisearch-http/src/index/mod.rs | 7 ++--- meilisearch-http/src/index/updates.rs | 26 +++++++++++++++---- .../src/index_controller/index_actor/actor.rs | 4 +-- .../index_actor/handle_impl.rs | 4 +-- .../index_controller/index_actor/message.rs | 4 +-- .../src/index_controller/index_actor/mod.rs | 6 ++--- meilisearch-http/src/index_controller/mod.rs | 6 ++--- .../src/index_controller/updates.rs | 4 +-- meilisearch-http/src/routes/settings/mod.rs | 7 ++--- 11 files changed, 47 insertions(+), 29 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index ec64192f7..c7979210e 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use sha2::Digest; -use crate::index::Settings; +use crate::index::{Checked, Settings}; use crate::index_controller::{IndexController, IndexStats, Stats}; use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::option::Opt; @@ -74,7 +74,7 @@ impl Data { Ok(Data { inner }) } - pub async fn settings(&self, uid: String) -> anyhow::Result { + pub async fn settings(&self, uid: String) -> anyhow::Result> { self.index_controller.settings(uid).await } diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index 518f9fcc3..23949aa86 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -2,7 +2,7 @@ use actix_web::web::Payload; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use super::Data; -use crate::index::Settings; +use crate::index::{Checked, Settings}; use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus}; impl Data { @@ -24,7 +24,7 @@ impl Data { pub async fn update_settings( &self, index: String, - settings: Settings, + settings: Settings, create: bool, ) -> anyhow::Result { let update = self diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 43c8b0193..c897fac3f 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashSet}; +use std::{collections::{BTreeSet, HashSet}, marker::PhantomData}; use std::ops::Deref; use std::sync::Arc; @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; use crate::helpers::EnvSizer; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; -pub use updates::{Facets, Settings}; +pub use updates::{Facets, Settings, Checked, Unchecked}; mod search; mod updates; @@ -27,7 +27,7 @@ impl Deref for Index { } impl Index { - pub fn settings(&self) -> anyhow::Result { + pub fn settings(&self) -> anyhow::Result> { let txn = self.read_txn()?; let displayed_attributes = self @@ -68,6 +68,7 @@ impl Index { ranking_rules: Some(Some(criteria)), stop_words: Some(Some(stop_words)), distinct_attribute: Some(distinct_attribute), + _kind: PhantomData, }) } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 4afd2e297..c35f21950 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeSet, HashMap}; use std::io; use std::num::NonZeroUsize; +use std::marker::PhantomData; use flate2::read::GzDecoder; use log::info; @@ -10,10 +11,15 @@ use serde::{de::Deserializer, Deserialize, Serialize}; use super::Index; use crate::index_controller::UpdateResult; +#[derive(Clone, Default, Debug)] +pub struct Checked; +#[derive(Clone, Default, Debug)] +pub struct Unchecked; + #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] -pub struct Settings { +pub struct Settings { #[serde( default, deserialize_with = "deserialize_some", @@ -49,21 +55,31 @@ pub struct Settings { skip_serializing_if = "Option::is_none" )] pub distinct_attribute: Option>, + + #[serde(skip)] + pub _kind: PhantomData, } -impl Settings { - pub fn cleared() -> Self { - Self { +impl Settings { + pub fn cleared() -> Settings { + Settings { displayed_attributes: Some(None), searchable_attributes: Some(None), attributes_for_faceting: Some(None), ranking_rules: Some(None), stop_words: Some(None), distinct_attribute: Some(None), + _kind: PhantomData, } } } +impl Settings { + pub fn check(self) -> Settings { + todo!() + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] @@ -137,7 +153,7 @@ impl Index { pub fn update_settings( &self, - settings: &Settings, + settings: &Settings, update_builder: UpdateBuilder, ) -> anyhow::Result { // We must use the write transaction of the update here. diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 06d2b0f60..1f1cf146b 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -10,7 +10,7 @@ use tokio::sync::mpsc; use tokio::task::spawn_blocking; use uuid::Uuid; -use crate::index::{Document, SearchQuery, SearchResult, Settings}; +use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::index_controller::{ get_arc_ownership_blocking, update_handler::UpdateHandler, Failed, IndexStats, Processed, Processing, @@ -164,7 +164,7 @@ impl IndexActor { .map_err(|e| IndexError::Error(e.into())) } - async fn handle_settings(&self, uuid: Uuid) -> IndexResult { + async fn handle_settings(&self, uuid: Uuid) -> IndexResult> { let index = self .store .get(uuid) diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 719e12cee..4569ea020 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use crate::index_controller::{IndexSettings, IndexStats, Processing}; +use crate::{index::Checked, index_controller::{IndexSettings, IndexStats, Processing}}; use crate::{ index::{Document, SearchQuery, SearchResult, Settings}, index_controller::{Failed, Processed}, @@ -57,7 +57,7 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn settings(&self, uuid: Uuid) -> IndexResult { + async fn settings(&self, uuid: Uuid) -> IndexResult> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Settings { uuid, ret }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index d728b2564..4e2824871 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use tokio::sync::oneshot; use uuid::Uuid; -use crate::index::{Document, SearchQuery, SearchResult, Settings}; +use crate::index::{Document, SearchQuery, SearchResult, Settings, Checked}; use crate::index_controller::{Failed, IndexStats, Processed, Processing}; use super::{IndexMeta, IndexResult, IndexSettings}; @@ -27,7 +27,7 @@ pub enum IndexMsg { }, Settings { uuid: Uuid, - ret: oneshot::Sender>, + ret: oneshot::Sender>>, }, Documents { uuid: Uuid, diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index ef5ea524c..f7f230349 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -14,7 +14,7 @@ pub use handle_impl::IndexActorHandleImpl; use message::IndexMsg; use store::{IndexStore, MapIndexStore}; -use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; +use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings}; use crate::index_controller::{Failed, Processed, Processing, IndexStats}; use super::IndexSettings; @@ -74,7 +74,7 @@ pub trait IndexActorHandle { data: Option, ) -> anyhow::Result>; async fn search(&self, uuid: Uuid, query: SearchQuery) -> IndexResult; - async fn settings(&self, uuid: Uuid) -> IndexResult; + async fn settings(&self, uuid: Uuid) -> IndexResult>; async fn documents( &self, @@ -130,7 +130,7 @@ mod test { self.as_ref().search(uuid, query).await } - async fn settings(&self, uuid: Uuid) -> IndexResult { + async fn settings(&self, uuid: Uuid) -> IndexResult> { self.as_ref().settings(uuid).await } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 81e6d0a5e..f1da36740 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -20,7 +20,7 @@ use snapshot::{SnapshotService, load_snapshot}; use update_actor::UpdateActorHandle; use uuid_resolver::{UuidError, UuidResolverHandle}; -use crate::index::{Settings, Document, SearchQuery, SearchResult}; +use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; mod index_actor; @@ -202,7 +202,7 @@ impl IndexController { pub async fn update_settings( &self, uid: String, - settings: Settings, + settings: Settings, create: bool, ) -> anyhow::Result { let perform_udpate = |uuid| async move { @@ -282,7 +282,7 @@ impl IndexController { Ok(ret) } - pub async fn settings(&self, uid: String) -> anyhow::Result { + pub async fn settings(&self, uid: String) -> anyhow::Result> { let uuid = self.uuid_resolver.get(uid.clone()).await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 02827abd0..a129a25a0 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; -use crate::index::Settings; +use crate::index::{Checked, Settings}; pub type UpdateError = String; @@ -25,7 +25,7 @@ pub enum UpdateMeta { }, ClearDocuments, DeleteDocuments, - Settings(Settings), + Settings(Settings), } #[derive(Debug, Serialize, Deserialize, Clone)] diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 236c94fed..03f1ee95c 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -1,6 +1,6 @@ use actix_web::{delete, get, post, web, HttpResponse}; -use crate::error::ResponseError; +use crate::{error::ResponseError, index::Unchecked}; use crate::helpers::Authentication; use crate::index::Settings; use crate::Data; @@ -138,10 +138,11 @@ create_services!( async fn update_all( data: web::Data, index_uid: web::Path, - body: web::Json, + body: web::Json>, ) -> Result { + let settings = body.into_inner().check(); match data - .update_settings(index_uid.into_inner(), body.into_inner(), true) + .update_settings(index_uid.into_inner(), settings, true) .await { Ok(update_result) => Ok( From 8d11b368d1595cfade894a29fbc3101410eea55f Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 10 May 2021 18:22:41 +0200 Subject: [PATCH 308/522] implement check --- meilisearch-http/src/index/updates.rs | 34 +++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index c35f21950..44d56ac51 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -75,8 +75,38 @@ impl Settings { } impl Settings { - pub fn check(self) -> Settings { - todo!() + pub fn check(mut self) -> Settings { + let displayed_attributes = match self.displayed_attributes.take() { + Some(Some(fields)) => { + if fields.iter().any(|f| f == "*") { + Some(None) + } else { + Some(Some(fields)) + } + } + otherwise => otherwise, + }; + + let searchable_attributes = match self.searchable_attributes.take() { + Some(Some(fields)) => { + if fields.iter().any(|f| f == "*") { + Some(None) + } else { + Some(Some(fields)) + } + } + otherwise => otherwise, + }; + + Settings { + displayed_attributes, + searchable_attributes, + attributes_for_faceting: self.attributes_for_faceting, + ranking_rules: self.ranking_rules, + stop_words: self.stop_words, + distinct_attribute: self.distinct_attribute, + _kind: PhantomData, + } } } From 0cc79d414f41d28870fab8543c23a454d18d7ead Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 10 May 2021 18:34:25 +0200 Subject: [PATCH 309/522] add test --- meilisearch-http/src/index/updates.rs | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 44d56ac51..0d76f2ae6 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -268,3 +268,42 @@ impl Index { } } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_setting_check() { + // test no changes + let settings = Settings { + displayed_attributes: Some(Some(vec![String::from("hello")])), + searchable_attributes: Some(Some(vec![String::from("hello")])), + attributes_for_faceting: None, + ranking_rules: None, + stop_words: None, + distinct_attribute: None, + _kind: PhantomData::, + }; + + let checked = settings.clone().check(); + assert_eq!(settings.displayed_attributes, checked.displayed_attributes); + assert_eq!(settings.searchable_attributes, checked.searchable_attributes); + + // test wildcard + // test no changes + let settings = Settings { + displayed_attributes: Some(Some(vec![String::from("*")])), + searchable_attributes: Some(Some(vec![String::from("hello"), String::from("*")])), + attributes_for_faceting: None, + ranking_rules: None, + stop_words: None, + distinct_attribute: None, + _kind: PhantomData::, + }; + + let checked = settings.check(); + assert_eq!(checked.displayed_attributes, Some(None)); + assert_eq!(checked.searchable_attributes, Some(None)); + } +} From e389c088ebb729865d3d93230852c8cfdac28b1a Mon Sep 17 00:00:00 2001 From: tamo Date: Wed, 28 Apr 2021 16:43:49 +0200 Subject: [PATCH 310/522] WIP: rebasing on master --- meilisearch-http/src/dump.rs | 423 ------------------ meilisearch-http/src/index_controller/dump.rs | 258 +++++++++++ .../src/index_controller/index_actor/actor.rs | 36 +- .../index_actor/handle_impl.rs | 9 +- .../index_controller/index_actor/message.rs | 5 + .../src/index_controller/index_actor/mod.rs | 1 - meilisearch-http/src/index_controller/mod.rs | 9 +- .../index_controller/update_actor/actor.rs | 93 +++- .../update_actor/handle_impl.rs | 14 + .../index_controller/update_actor/message.rs | 9 + .../src/index_controller/update_actor/mod.rs | 5 +- .../update_actor/update_store.rs | 47 ++ .../index_controller/uuid_resolver/actor.rs | 7 + .../uuid_resolver/handle_impl.rs | 10 + .../index_controller/uuid_resolver/message.rs | 4 + .../src/index_controller/uuid_resolver/mod.rs | 1 + .../index_controller/uuid_resolver/store.rs | 28 ++ meilisearch-http/src/option.rs | 1 + meilisearch-http/src/routes/index.rs | 11 + .../tests/settings/get_settings.rs | 2 +- 20 files changed, 540 insertions(+), 433 deletions(-) delete mode 100644 meilisearch-http/src/dump.rs create mode 100644 meilisearch-http/src/index_controller/dump.rs diff --git a/meilisearch-http/src/dump.rs b/meilisearch-http/src/dump.rs deleted file mode 100644 index 544fffaa7..000000000 --- a/meilisearch-http/src/dump.rs +++ /dev/null @@ -1,423 +0,0 @@ -use std::fs::{create_dir_all, File}; -use std::io::prelude::*; -use std::path::{Path, PathBuf}; -use std::sync::Mutex; -use std::thread; - -use actix_web::web; -use chrono::offset::Utc; -use indexmap::IndexMap; -use log::{error, info}; -use once_cell::sync::Lazy; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tempfile::TempDir; - -use crate::Data; -use crate::error::{Error, ResponseError}; -use crate::helpers::compression; -use crate::routes::index; -use crate::routes::setting::Settings; -use crate::routes::index::IndexResponse; - -// Mutex to share dump progress. -static DUMP_INFO: Lazy>> = Lazy::new(Mutex::default); - -#[derive(Debug, Serialize, Deserialize, Copy, Clone)] -enum DumpVersion { - V1, -} - -impl DumpVersion { - const CURRENT: Self = Self::V1; -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DumpMetadata { - indexes: Vec, - db_version: String, - dump_version: DumpVersion, -} - -impl DumpMetadata { - /// Create a DumpMetadata with the current dump version of meilisearch. - pub fn new(indexes: Vec, db_version: String) -> Self { - DumpMetadata { - indexes, - db_version, - dump_version: DumpVersion::CURRENT, - } - } - - /// Extract DumpMetadata from `metadata.json` file present at provided `dir_path` - fn from_path(dir_path: &Path) -> Result { - let path = dir_path.join("metadata.json"); - let file = File::open(path)?; - let reader = std::io::BufReader::new(file); - let metadata = serde_json::from_reader(reader)?; - - Ok(metadata) - } - - /// Write DumpMetadata in `metadata.json` file at provided `dir_path` - fn to_path(&self, dir_path: &Path) -> Result<(), Error> { - let path = dir_path.join("metadata.json"); - let file = File::create(path)?; - - serde_json::to_writer(file, &self)?; - - Ok(()) - } -} - -/// Extract Settings from `settings.json` file present at provided `dir_path` -fn settings_from_path(dir_path: &Path) -> Result { - let path = dir_path.join("settings.json"); - let file = File::open(path)?; - let reader = std::io::BufReader::new(file); - let metadata = serde_json::from_reader(reader)?; - - Ok(metadata) -} - -/// Write Settings in `settings.json` file at provided `dir_path` -fn settings_to_path(settings: &Settings, dir_path: &Path) -> Result<(), Error> { - let path = dir_path.join("settings.json"); - let file = File::create(path)?; - - serde_json::to_writer(file, settings)?; - - Ok(()) -} - -/// Import settings and documents of a dump with version `DumpVersion::V1` in specified index. -fn import_index_v1( - data: &Data, - dumps_dir: &Path, - index_uid: &str, - document_batch_size: usize, - write_txn: &mut MainWriter, -) -> Result<(), Error> { - - // open index - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - // index dir path in dump dir - let index_path = &dumps_dir.join(index_uid); - - // extract `settings.json` file and import content - let settings = settings_from_path(&index_path)?; - let settings = settings.to_update().map_err(|e| Error::dump_failed(format!("importing settings for index {}; {}", index_uid, e)))?; - apply_settings_update(write_txn, &index, settings)?; - - // create iterator over documents in `documents.jsonl` to make batch importation - // create iterator over documents in `documents.jsonl` to make batch importation - let documents = { - let file = File::open(&index_path.join("documents.jsonl"))?; - let reader = std::io::BufReader::new(file); - let deserializer = serde_json::Deserializer::from_reader(reader); - deserializer.into_iter::>() - }; - - // batch import document every `document_batch_size`: - // create a Vec to bufferize documents - let mut values = Vec::with_capacity(document_batch_size); - // iterate over documents - for document in documents { - // push document in buffer - values.push(document?); - // if buffer is full, create and apply a batch, and clean buffer - if values.len() == document_batch_size { - let batch = std::mem::replace(&mut values, Vec::with_capacity(document_batch_size)); - apply_documents_addition(write_txn, &index, batch)?; - } - } - - // apply documents remaining in the buffer - if !values.is_empty() { - apply_documents_addition(write_txn, &index, values)?; - } - - // sync index information: stats, updated_at, last_update - if let Err(e) = crate::index_update_callback_txn(index, index_uid, data, write_txn) { - return Err(Error::Internal(e)); - } - - Ok(()) -} - -/// Import dump from `dump_path` in database. -pub fn import_dump( - data: &Data, - dump_path: &Path, - document_batch_size: usize, -) -> Result<(), Error> { - info!("Importing dump from {:?}...", dump_path); - - // create a temporary directory - let tmp_dir = TempDir::new()?; - let tmp_dir_path = tmp_dir.path(); - - // extract dump in temporary directory - compression::from_tar_gz(dump_path, tmp_dir_path)?; - - // read dump metadata - let metadata = DumpMetadata::from_path(&tmp_dir_path)?; - - // choose importation function from DumpVersion of metadata - let import_index = match metadata.dump_version { - DumpVersion::V1 => import_index_v1, - }; - - // remove indexes which have same `uid` than indexes to import and create empty indexes - let existing_index_uids = data.db.indexes_uids(); - for index in metadata.indexes.iter() { - if existing_index_uids.contains(&index.uid) { - data.db.delete_index(index.uid.clone())?; - } - index::create_index_sync(&data.db, index.uid.clone(), index.name.clone(), index.primary_key.clone())?; - } - - // import each indexes content - data.db.main_write::<_, _, Error>(|mut writer| { - for index in metadata.indexes { - import_index(&data, tmp_dir_path, &index.uid, document_batch_size, &mut writer)?; - } - Ok(()) - })?; - - info!("Dump importation from {:?} succeed", dump_path); - Ok(()) -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] -#[serde(rename_all = "snake_case")] -pub enum DumpStatus { - Done, - InProgress, - Failed, -} - -#[derive(Debug, Serialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct DumpInfo { - pub uid: String, - pub status: DumpStatus, - #[serde(skip_serializing_if = "Option::is_none", flatten)] - pub error: Option, -} - -impl DumpInfo { - pub fn new(uid: String, status: DumpStatus) -> Self { - Self { uid, status, error: None } - } - - pub fn with_error(mut self, error: ResponseError) -> Self { - self.status = DumpStatus::Failed; - self.error = Some(json!(error)); - - self - } - - pub fn dump_already_in_progress(&self) -> bool { - self.status == DumpStatus::InProgress - } - - pub fn get_current() -> Option { - DUMP_INFO.lock().unwrap().clone() - } - - pub fn set_current(&self) { - *DUMP_INFO.lock().unwrap() = Some(self.clone()); - } -} - -/// Generate uid from creation date -fn generate_uid() -> String { - Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() -} - -/// Infer dumps_dir from dump_uid -pub fn compressed_dumps_dir(dumps_dir: &Path, dump_uid: &str) -> PathBuf { - dumps_dir.join(format!("{}.dump", dump_uid)) -} - -/// Write metadata in dump -fn dump_metadata(data: &web::Data, dir_path: &Path, indexes: Vec) -> Result<(), Error> { - let (db_major, db_minor, db_patch) = data.db.version(); - let metadata = DumpMetadata::new(indexes, format!("{}.{}.{}", db_major, db_minor, db_patch)); - - metadata.to_path(dir_path) -} - -/// Export settings of provided index in dump -fn dump_index_settings(data: &web::Data, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { - let settings = crate::routes::setting::get_all_sync(data, reader, index_uid)?; - - settings_to_path(&settings, dir_path) -} - -/// Export updates of provided index in dump -fn dump_index_updates(data: &web::Data, reader: &UpdateReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { - let updates_path = dir_path.join("updates.jsonl"); - let updates = crate::routes::index::get_all_updates_status_sync(data, reader, index_uid)?; - - let file = File::create(updates_path)?; - - for update in updates { - serde_json::to_writer(&file, &update)?; - writeln!(&file)?; - } - - Ok(()) -} - -/// Export documents of provided index in dump -fn dump_index_documents(data: &web::Data, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { - let documents_path = dir_path.join("documents.jsonl"); - let file = File::create(documents_path)?; - let dump_batch_size = data.dump_batch_size; - - let mut offset = 0; - loop { - let documents = crate::routes::document::get_all_documents_sync(data, reader, index_uid, offset, dump_batch_size, None)?; - if documents.is_empty() { break; } else { offset += dump_batch_size; } - - for document in documents { - serde_json::to_writer(&file, &document)?; - writeln!(&file)?; - } - } - - Ok(()) -} - -/// Write error with a context. -fn fail_dump_process(dump_info: DumpInfo, context: &str, error: E) { - let error_message = format!("{}; {}", context, error); - - error!("Something went wrong during dump process: {}", &error_message); - dump_info.with_error(Error::dump_failed(error_message).into()).set_current(); -} - -/// Main function of dump. -fn dump_process(data: web::Data, dumps_dir: PathBuf, dump_info: DumpInfo) { - // open read transaction on Update - let update_reader = match data.db.update_read_txn() { - Ok(r) => r, - Err(e) => { - fail_dump_process(dump_info, "creating RO transaction on updates", e); - return ; - } - }; - - // open read transaction on Main - let main_reader = match data.db.main_read_txn() { - Ok(r) => r, - Err(e) => { - fail_dump_process(dump_info, "creating RO transaction on main", e); - return ; - } - }; - - // create a temporary directory - let tmp_dir = match TempDir::new() { - Ok(tmp_dir) => tmp_dir, - Err(e) => { - fail_dump_process(dump_info, "creating temporary directory", e); - return ; - } - }; - let tmp_dir_path = tmp_dir.path(); - - // fetch indexes - let indexes = match crate::routes::index::list_indexes_sync(&data, &main_reader) { - Ok(indexes) => indexes, - Err(e) => { - fail_dump_process(dump_info, "listing indexes", e); - return ; - } - }; - - // create metadata - if let Err(e) = dump_metadata(&data, &tmp_dir_path, indexes.clone()) { - fail_dump_process(dump_info, "generating metadata", e); - return ; - } - - // export settings, updates and documents for each indexes - for index in indexes { - let index_path = tmp_dir_path.join(&index.uid); - - // create index sub-dircetory - if let Err(e) = create_dir_all(&index_path) { - fail_dump_process(dump_info, &format!("creating directory for index {}", &index.uid), e); - return ; - } - - // export settings - if let Err(e) = dump_index_settings(&data, &main_reader, &index_path, &index.uid) { - fail_dump_process(dump_info, &format!("generating settings for index {}", &index.uid), e); - return ; - } - - // export documents - if let Err(e) = dump_index_documents(&data, &main_reader, &index_path, &index.uid) { - fail_dump_process(dump_info, &format!("generating documents for index {}", &index.uid), e); - return ; - } - - // export updates - if let Err(e) = dump_index_updates(&data, &update_reader, &index_path, &index.uid) { - fail_dump_process(dump_info, &format!("generating updates for index {}", &index.uid), e); - return ; - } - } - - // compress dump in a file named `{dump_uid}.dump` in `dumps_dir` - if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_dir(&dumps_dir, &dump_info.uid)) { - fail_dump_process(dump_info, "compressing dump", e); - return ; - } - - // update dump info to `done` - let resume = DumpInfo::new( - dump_info.uid, - DumpStatus::Done - ); - - resume.set_current(); -} - -pub fn init_dump_process(data: &web::Data, dumps_dir: &Path) -> Result { - create_dir_all(dumps_dir).map_err(|e| Error::dump_failed(format!("creating temporary directory {}", e)))?; - - // check if a dump is already in progress - if let Some(resume) = DumpInfo::get_current() { - if resume.dump_already_in_progress() { - return Err(Error::dump_conflict()) - } - } - - // generate a new dump info - let info = DumpInfo::new( - generate_uid(), - DumpStatus::InProgress - ); - - info.set_current(); - - let data = data.clone(); - let dumps_dir = dumps_dir.to_path_buf(); - let info_cloned = info.clone(); - // run dump process in a new thread - thread::spawn(move || - dump_process(data, dumps_dir, info_cloned) - ); - - Ok(info) -} diff --git a/meilisearch-http/src/index_controller/dump.rs b/meilisearch-http/src/index_controller/dump.rs new file mode 100644 index 000000000..afdcfd9ce --- /dev/null +++ b/meilisearch-http/src/index_controller/dump.rs @@ -0,0 +1,258 @@ +use std::{ + fs::File, + path::{Path, PathBuf}, + sync::Arc, +}; + +use anyhow::bail; +use heed::EnvOpenOptions; +use log::{error, info}; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use serde::{Deserialize, Serialize}; +use tempfile::TempDir; +use tokio::fs; +use tokio::task::spawn_blocking; + +use super::update_actor::UpdateActorHandle; +use super::uuid_resolver::UuidResolverHandle; +use super::IndexMetadata; +use crate::index::Index; +use crate::index_controller::uuid_resolver; +use crate::{helpers::compression, index::Settings}; + +#[derive(Debug, Serialize, Deserialize, Copy, Clone)] +enum DumpVersion { + V1, +} + +impl DumpVersion { + const CURRENT: Self = Self::V1; +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DumpMetadata { + indexes: Vec, + db_version: String, + dump_version: DumpVersion, +} + +impl DumpMetadata { + /// Create a DumpMetadata with the current dump version of meilisearch. + pub fn new(indexes: Vec, db_version: String) -> Self { + DumpMetadata { + indexes, + db_version, + dump_version: DumpVersion::CURRENT, + } + } + + /// Extract DumpMetadata from `metadata.json` file present at provided `dir_path` + fn from_path(dir_path: &Path) -> anyhow::Result { + let path = dir_path.join("metadata.json"); + let file = File::open(path)?; + let reader = std::io::BufReader::new(file); + let metadata = serde_json::from_reader(reader)?; + + Ok(metadata) + } + + /// Write DumpMetadata in `metadata.json` file at provided `dir_path` + fn to_path(&self, dir_path: &Path) -> anyhow::Result<()> { + let path = dir_path.join("metadata.json"); + let file = File::create(path)?; + + serde_json::to_writer(file, &self)?; + + Ok(()) + } +} + +pub struct DumpService { + uuid_resolver_handle: R, + update_handle: U, + dump_path: PathBuf, + db_name: String, +} + +impl DumpService +where + U: UpdateActorHandle, + R: UuidResolverHandle, +{ + pub fn new( + uuid_resolver_handle: R, + update_handle: U, + dump_path: PathBuf, + db_name: String, + ) -> Self { + Self { + uuid_resolver_handle, + update_handle, + dump_path, + db_name, + } + } + + pub async fn run(self) { + if let Err(e) = self.perform_dump().await { + error!("{}", e); + } + } + + async fn perform_dump(&self) -> anyhow::Result<()> { + info!("Performing dump."); + + let dump_dir = self.dump_path.clone(); + fs::create_dir_all(&dump_dir).await?; + let temp_dump_dir = spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let uuids = self + .uuid_resolver_handle + .dump(temp_dump_path.clone()) + .await?; + + if uuids.is_empty() { + return Ok(()); + } + + let tasks = uuids + .iter() + .map(|&uuid| self.update_handle.dump(uuid, temp_dump_path.clone())) + .collect::>(); + + futures::future::try_join_all(tasks).await?; + + let dump_dir = self.dump_path.clone(); + let dump_path = self.dump_path.join(format!("{}.dump", self.db_name)); + let dump_path = spawn_blocking(move || -> anyhow::Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; + let temp_dump_file_path = temp_dump_file.path().to_owned(); + compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; + temp_dump_file.persist(&dump_path)?; + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(()) + } +} + +/// Extract Settings from `settings.json` file present at provided `dir_path` +fn settings_from_path(dir_path: &Path) -> anyhow::Result { + let path = dir_path.join("settings.json"); + let file = File::open(path)?; + let reader = std::io::BufReader::new(file); + let metadata = serde_json::from_reader(reader)?; + + Ok(metadata) +} + +/// Write Settings in `settings.json` file at provided `dir_path` +fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> { + let path = dir_path.join("settings.json"); + let file = File::create(path)?; + + serde_json::to_writer(file, settings)?; + + Ok(()) +} + +fn import_index_v1(size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(&index_path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, index_path)?; + let index = Index(Arc::new(index)); + + // extract `settings.json` file and import content + let settings = settings_from_path(&dump_path)?; + let update_builder = UpdateBuilder::new(0); + index.update_settings(&settings, update_builder)?; + + let update_builder = UpdateBuilder::new(1); + let file = File::open(&index_path.join("documents.jsonl"))?; + let reader = std::io::BufReader::new(file); + index.update_documents( + UpdateFormat::JsonStream, + IndexDocumentsMethod::ReplaceDocuments, + reader, + update_builder, + None, + )?; + + // the last step: we extract the milli::Index and close it + Arc::try_unwrap(index.0) + .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") + .unwrap() + .prepare_for_closing() + .wait(); + + Ok(()) +} + +pub fn load_dump( + db_path: impl AsRef, + dump_path: impl AsRef, + size: usize, +) -> anyhow::Result<()> { + info!("Importing dump from {}...", dump_path.as_ref().display()); + let db_path = db_path.as_ref(); + let dump_path = dump_path.as_ref(); + let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&db_path)?; + + // extract the dump in a temporary directory + let tmp_dir = TempDir::new()?; + let tmp_dir_path = tmp_dir.path(); + compression::from_tar_gz(dump_path, tmp_dir_path)?; + + // read dump metadata + let metadata = DumpMetadata::from_path(&tmp_dir_path)?; + + // choose importation function from DumpVersion of metadata + let import_index = match metadata.dump_version { + DumpVersion::V1 => import_index_v1, + }; + + // remove indexes which have same `uuid` than indexes to import and create empty indexes + let existing_index_uids = futures::executor::block_on(uuid_resolver.list())?; + + info!("Deleting indexes provided in the dump..."); + for idx in &metadata.indexes { + if let Some((_, uuid)) = existing_index_uids.iter().find(|(s, _)| s == &idx.uid) { + // if we find the index in the `uuid_resolver` it's supposed to exist on the file system + // and we want to delete it + let path = db_path.join(&format!("indexes/index-{}", uuid)); + info!("Deleting {}", path.display()); + use std::io::ErrorKind::*; + match std::fs::remove_dir_all(path) { + Ok(()) => (), + // if an index was present in the metadata but missing of the fs we can ignore the + // problem because we are going to create it later + Err(e) if e.kind() == NotFound => (), + Err(e) => bail!(e), + } + } else { + // if the index does not exist in the `uuid_resolver` we create it + futures::executor::block_on(uuid_resolver.create(idx.uid.clone()))?; + } + } + + // import each indexes content + for idx in metadata.indexes { + let dump_path = tmp_dir_path.join(&idx.uid); + let uuid = futures::executor::block_on(uuid_resolver.get(idx.uid))?; + let index_path = db_path.join(&format!("indexes/index-{}", uuid)); + + info!("Importing dump from {} into {}...", dump_path.display(), index_path.display()); + import_index(size, &dump_path, &index_path).unwrap(); + info!("Dump importation from {} succeed", dump_path.display()); + } + + + info!("Dump importation from {} succeed", dump_path.display()); + Ok(()) +} diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 1f1cf146b..535c405dc 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -36,6 +36,9 @@ impl IndexActor { Ok(Self { receiver, update_handler, store }) } + /// `run` poll the write_receiver and read_receiver concurrently, but while messages send + /// through the read channel are processed concurrently, the messages sent through the write + /// channel are processed one at a time. pub async fn run(mut self) { let mut receiver = self .receiver @@ -119,6 +122,9 @@ impl IndexActor { Snapshot { uuid, path, ret } => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } + Dump { uuid, path, ret } => { + let _ = ret.send(self.handle_dump(uuid, path).await); + } GetStats { uuid, ret } => { let _ = ret.send(self.handle_get_stats(uuid).await); } @@ -306,7 +312,35 @@ impl IndexActor { Ok(()) } - async fn handle_get_stats(&self, uuid: Uuid) -> IndexResult { + async fn handle_dump(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> { + use tokio::fs::create_dir_all; + + path.push("indexes"); + create_dir_all(&path) + .await + .map_err(|e| IndexError::Error(e.into()))?; + + if let Some(index) = self.store.get(uuid).await? { + let mut index_path = path.join(format!("index-{}", uuid)); + create_dir_all(&index_path) + .await + .map_err(|e| IndexError::Error(e.into()))?; + index_path.push("data.mdb"); + spawn_blocking(move || -> anyhow::Result<()> { + // Get write txn to wait for ongoing write transaction before dump. + let _txn = index.write_txn()?; + index.env.copy_to_path(index_path, CompactionOption::Enabled)?; + Ok(()) + }) + .await + .map_err(|e| IndexError::Error(e.into()))? + .map_err(IndexError::Error)?; + } + + Ok(()) + } + + async fn handle_get_stats(&self, uuid: Uuid) -> Result { let index = self .store .get(uuid) diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 4569ea020..d625a763e 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -136,7 +136,14 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Dump { uuid, path, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn get_index_stats(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetStats { uuid, ret }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 4e2824871..0d88532ca 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -60,6 +60,11 @@ pub enum IndexMsg { path: PathBuf, ret: oneshot::Sender>, }, + Dump { + uuid: Uuid, + path: PathBuf, + ret: oneshot::Sender>, + }, GetStats { uuid: Uuid, ret: oneshot::Sender>, diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index f7f230349..46105742b 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -180,5 +180,4 @@ mod test { async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { self.as_ref().get_index_stats(uuid).await } - } } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index f1da36740..10b9142cc 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -5,7 +5,6 @@ use std::time::Duration; use actix_web::web::{Bytes, Payload}; use anyhow::bail; -use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use log::info; use milli::FieldsDistribution; @@ -25,6 +24,7 @@ use crate::option::Opt; mod index_actor; mod snapshot; +mod dump; mod update_actor; mod update_handler; mod updates; @@ -87,6 +87,13 @@ impl IndexController { options.ignore_snapshot_if_db_exists, options.ignore_missing_snapshot, )?; + } else if let Some(ref path) = options.import_dump { + load_dump( + &options.db_path, + path, + index_size, + ); + } std::fs::create_dir_all(&path)?; diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index e47edc5bc..7885d0b3b 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -71,11 +71,16 @@ where Some(Delete { uuid, ret }) => { let _ = ret.send(self.handle_delete(uuid).await); } - Some(Snapshot { uuids, path, ret }) => { - let _ = ret.send(self.handle_snapshot(uuids, path).await); + Some(Snapshot { uuid, path, ret }) => { + let _ = ret.send(self.handle_snapshot(uuid, path).await); + } + Some(Dump { uuid, path, ret }) => { + let _ = ret.send(self.handle_dump(uuid, path).await); } Some(GetInfo { ret }) => { let _ = ret.send(self.handle_get_info().await); + Some(GetSize { uuid, ret }) => { + let _ = ret.send(self.handle_get_size(uuid).await); } None => break, } @@ -194,9 +199,51 @@ where } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let store = self.store.clone(); + let store = self.store.delete(uuid).await?; - tokio::task::spawn_blocking(move || store.delete_all(uuid)) + if let Some(store) = store { + tokio::task::spawn(async move { + let store = get_arc_ownership_blocking(store).await; + tokio::task::spawn_blocking(move || { + store.prepare_for_closing().wait(); + info!("Update store {} was closed.", uuid); + }); + }); + } + + Ok(()) + } + + async fn handle_create(&self, uuid: Uuid) -> Result<()> { + let _ = self.store.get_or_create(uuid).await?; + Ok(()) + } + + Ok(()) + } + + async fn handle_create(&self, uuid: Uuid) -> Result<()> { + let _ = self.store.get_or_create(uuid).await?; + Ok(()) + } + + async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let index_handle = self.index_handle.clone(); + if let Some(update_store) = self.store.get(uuid).await? { + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + // acquire write lock to prevent further writes during snapshot + // the update lock must be acquired BEFORE the write lock to prevent dead lock + let _lock = update_store.update_lock.lock(); + let mut txn = update_store.env.write_txn()?; + + // create db snapshot + update_store.snapshot(&mut txn, &path, uuid)?; + + futures::executor::block_on( + async move { index_handle.snapshot(uuid, path).await }, + )?; + Ok(()) + }) .await .map_err(|e| UpdateError::Error(e.into()))? .map_err(|e| UpdateError::Error(e.into()))?; @@ -245,4 +292,42 @@ where Ok(info) } + + async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let index_handle = self.index_handle.clone(); + if let Some(update_store) = self.store.get(uuid).await? { + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + // acquire write lock to prevent further writes during the dump + // the update lock must be acquired BEFORE the write lock to prevent dead lock + let _lock = update_store.update_lock.lock(); + let mut txn = update_store.env.write_txn()?; + + // create db dump + update_store.dump(&mut txn, &path, uuid)?; + + futures::executor::block_on( + async move { index_handle.dump(uuid, path).await }, + )?; + Ok(()) + }) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?; + } + + Ok(()) + } + + async fn handle_get_size(&self, uuid: Uuid) -> Result { + let size = match self.store.get(uuid).await? { + Some(update_store) => tokio::task::spawn_blocking(move || -> anyhow::Result { + let txn = update_store.env.read_txn()?; + + update_store.get_size(&txn) + }) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?, + None => 0, + }; } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 999481573..569b896b0 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -78,6 +78,20 @@ where receiver.await.expect("update actor killed.") } + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Dump { uuid, path, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + + async fn get_size(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::GetSize { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 17b2b3579..3f39c224f 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -31,7 +31,16 @@ pub enum UpdateMsg { path: PathBuf, ret: oneshot::Sender>, }, + Dump { + uuid: Uuid, + path: PathBuf, + ret: oneshot::Sender>, + }, GetInfo { ret: oneshot::Sender>, }, + GetSize { + uuid: Uuid, + ret: oneshot::Sender>, + }, } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index e7a12b7ff..4d8ab6f20 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -40,8 +40,11 @@ pub trait UpdateActorHandle { async fn get_all_updates_status(&self, uuid: Uuid) -> Result>; async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; - async fn snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()>; + async fn create(&self, uuid: Uuid) -> Result<()>; + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>; async fn get_info(&self) -> Result; + async fn get_size(&self, uuid: Uuid) -> Result; async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 6a916af33..4bc4c8c75 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -499,9 +499,56 @@ impl UpdateStore { Ok(()) } + pub fn dump( + &self, + txn: &mut heed::RwTxn, + path: impl AsRef, + uuid: Uuid, + ) -> anyhow::Result<()> { + let update_path = path.as_ref().join("updates"); + create_dir_all(&update_path)?; + + let mut dump_path = update_path.join(format!("update-{}", uuid)); + // acquire write lock to prevent further writes during dump + create_dir_all(&dump_path)?; + dump_path.push("data.mdb"); + + // create db dump + self.env.copy_to_path(&dump_path, CompactionOption::Enabled)?; + + let update_files_path = update_path.join("update_files"); + create_dir_all(&update_files_path)?; + + for path in self.pending.iter(&txn)? { + let (_, path) = path?; + let name = path.file_name().unwrap(); + let to = update_files_path.join(name); + copy(path, to)?; + } + + Ok(()) + } + pub fn get_info(&self) -> anyhow::Result { let mut size = self.env.size(); let txn = self.env.read_txn()?; + for entry in self.pending_queue.iter(&txn)? { + let (_, pending) = entry?; + if let Some(path) = pending.content_path() { + size += File::open(path)?.metadata()?.len(); + } + } + let processing = match *self.state.read() { + State::Processing(uuid, _) => Some(uuid), + _ => None, + }; + + Ok(UpdateStoreInfo { size, processing }) + } + + pub fn get_size(&self, txn: &heed::RoTxn) -> anyhow::Result { + let mut size = self.env.size(); + let txn = self.env.read_txn()?; for entry in self.pending_queue.iter(&txn)? { let (_, pending) = entry?; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 253326276..9c180e4a8 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -41,6 +41,9 @@ impl UuidResolverActor { Some(SnapshotRequest { path, ret }) => { let _ = ret.send(self.handle_snapshot(path).await); } + Some(DumpRequest { path, ret }) => { + let _ = ret.send(self.handle_dump(path).await); + } Some(GetSize { ret }) => { let _ = ret.send(self.handle_get_size().await); } @@ -82,6 +85,10 @@ impl UuidResolverActor { self.store.snapshot(path).await } + async fn handle_dump(&self, path: PathBuf) -> Result> { + self.store.dump(path).await + } + async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> { if !is_index_uid_valid(&uid) { return Err(UuidError::BadlyFormatted(uid)); diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index db4c482bd..e47f9a8e0 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -68,6 +68,7 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } + /// TODO: we should merge this function with the dump function async fn snapshot(&self, path: PathBuf) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::SnapshotRequest { path, ret }; @@ -77,6 +78,15 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } + async fn dump(&self, path: PathBuf) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::DumpRequest { path, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + async fn get_size(&self) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::GetSize { ret }; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index a72bf0587..67493c2cd 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -31,6 +31,10 @@ pub enum UuidResolveMsg { path: PathBuf, ret: oneshot::Sender>>, }, + DumpRequest { + path: PathBuf, + ret: oneshot::Sender>>, + }, GetSize { ret: oneshot::Sender>, }, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index ef17133ff..a8361095c 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -31,6 +31,7 @@ pub trait UuidResolverHandle { async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; async fn snapshot(&self, path: PathBuf) -> Result>; + async fn dump(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 29c034c44..df4c3a2fb 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -21,6 +21,7 @@ pub trait UuidStore { async fn list(&self) -> Result>; async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; async fn snapshot(&self, path: PathBuf) -> Result>; + async fn dump(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } @@ -130,6 +131,8 @@ impl UuidStore for HeedUuidStore { .await? } + // TODO: we should merge this function and the following function for the dump. it's exactly + // the same code async fn snapshot(&self, mut path: PathBuf) -> Result> { let env = self.env.clone(); let db = self.db; @@ -155,6 +158,31 @@ impl UuidStore for HeedUuidStore { .await? } + async fn dump(&self, mut path: PathBuf) -> Result> { + let env = self.env.clone(); + let db = self.db; + tokio::task::spawn_blocking(move || { + // Write transaction to acquire a lock on the database. + let txn = env.write_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (_, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push(uuid) + } + + // only perform dump if there are indexes + if !entries.is_empty() { + path.push("index_uuids"); + create_dir_all(&path).unwrap(); + path.push("data.mdb"); + env.copy_to_path(path, CompactionOption::Enabled)?; + } + Ok(entries) + }) + .await? + } + async fn get_size(&self) -> Result { Ok(self.env.size()) } diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 1997718cc..87238c4d7 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -203,6 +203,7 @@ pub struct Opt { pub import_dump: Option, /// The batch size used in the importation process, the bigger it is the faster the dump is created. + /// This options is now deprecated and will be ignored #[structopt(long, env = "MEILI_DUMP_BATCH_SIZE", default_value = "1024")] pub dump_batch_size: usize, diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 4424c8cfe..1afc01806 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -1,5 +1,6 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; +use chrono::DateTime; use serde::Deserialize; use crate::error::ResponseError; @@ -68,6 +69,16 @@ struct UpdateIndexRequest { primary_key: Option, } +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UpdateIndexResponse { + name: String, + uid: String, + created_at: DateTime, + updated_at: DateTime, + primary_key: Option, +} + #[put("/indexes/{index_uid}", wrap = "Authentication::Private")] async fn update_index( data: web::Data, diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 4230e19f8..e5f51d7f0 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -19,7 +19,7 @@ async fn get_settings() { assert_eq!(settings.keys().len(), 6); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); - assert_eq!(settings["attributesForFaceting"], json!({})); + assert_eq!(settings["attributesForFaceting"], json!(null)); assert_eq!(settings["distinctAttribute"], json!(null)); assert_eq!( settings["rankingRules"], From c4d898a26545a06e01bc696f24d6e7b00198c1f8 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 27 Apr 2021 10:27:43 +0200 Subject: [PATCH 311/522] split the dumps between v1 and v2 --- meilisearch-http/src/data/mod.rs | 4 +- .../index_controller/{dump.rs => dump/mod.rs} | 91 +++++--------- .../src/index_controller/dump/v1.rs | 119 ++++++++++++++++++ .../src/index_controller/dump/v2.rs | 51 ++++++++ meilisearch-http/src/index_controller/mod.rs | 4 +- meilisearch-http/src/main.rs | 2 +- 6 files changed, 205 insertions(+), 66 deletions(-) rename meilisearch-http/src/index_controller/{dump.rs => dump/mod.rs} (73%) create mode 100644 meilisearch-http/src/index_controller/dump/v1.rs create mode 100644 meilisearch-http/src/index_controller/dump/v2.rs diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index c7979210e..e2bb7fbfb 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -55,10 +55,10 @@ impl ApiKeys { } impl Data { - pub fn new(options: Opt) -> anyhow::Result { + pub async fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - let index_controller = IndexController::new(&path, &options)?; + let index_controller = IndexController::new(&path, &options).await?; let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/meilisearch-http/src/index_controller/dump.rs b/meilisearch-http/src/index_controller/dump/mod.rs similarity index 73% rename from meilisearch-http/src/index_controller/dump.rs rename to meilisearch-http/src/index_controller/dump/mod.rs index afdcfd9ce..7278a7b77 100644 --- a/meilisearch-http/src/index_controller/dump.rs +++ b/meilisearch-http/src/index_controller/dump/mod.rs @@ -1,14 +1,13 @@ -use std::{ - fs::File, - path::{Path, PathBuf}, - sync::Arc, -}; +mod v1; +mod v2; + +use std::{fs::File, path::{Path, PathBuf}, sync::Arc}; use anyhow::bail; use heed::EnvOpenOptions; use log::{error, info}; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use serde::{Deserialize, Serialize}; +use serde::{de::Deserializer, Deserialize, Serialize}; use tempfile::TempDir; use tokio::fs; use tokio::task::spawn_blocking; @@ -20,13 +19,30 @@ use crate::index::Index; use crate::index_controller::uuid_resolver; use crate::{helpers::compression, index::Settings}; +pub (super) fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: Deserialize<'de>, + D: Deserializer<'de>, +{ + Deserialize::deserialize(deserializer).map(Some) +} + #[derive(Debug, Serialize, Deserialize, Copy, Clone)] enum DumpVersion { V1, + V2, } impl DumpVersion { - const CURRENT: Self = Self::V1; + const CURRENT: Self = Self::V2; + + /// Select the good importation function from the `DumpVersion` of metadata + pub fn import_index(self, size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { + match self { + Self::V1 => v1::import_index(size, dump_path, index_path), + Self::V2 => v2::import_index(size, dump_path, index_path), + } + } } #[derive(Debug, Serialize, Deserialize)] @@ -141,16 +157,6 @@ where } } -/// Extract Settings from `settings.json` file present at provided `dir_path` -fn settings_from_path(dir_path: &Path) -> anyhow::Result { - let path = dir_path.join("settings.json"); - let file = File::open(path)?; - let reader = std::io::BufReader::new(file); - let metadata = serde_json::from_reader(reader)?; - - Ok(metadata) -} - /// Write Settings in `settings.json` file at provided `dir_path` fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> { let path = dir_path.join("settings.json"); @@ -161,40 +167,7 @@ fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> Ok(()) } -fn import_index_v1(size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { - std::fs::create_dir_all(&index_path)?; - let mut options = EnvOpenOptions::new(); - options.map_size(size); - let index = milli::Index::new(options, index_path)?; - let index = Index(Arc::new(index)); - - // extract `settings.json` file and import content - let settings = settings_from_path(&dump_path)?; - let update_builder = UpdateBuilder::new(0); - index.update_settings(&settings, update_builder)?; - - let update_builder = UpdateBuilder::new(1); - let file = File::open(&index_path.join("documents.jsonl"))?; - let reader = std::io::BufReader::new(file); - index.update_documents( - UpdateFormat::JsonStream, - IndexDocumentsMethod::ReplaceDocuments, - reader, - update_builder, - None, - )?; - - // the last step: we extract the milli::Index and close it - Arc::try_unwrap(index.0) - .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") - .unwrap() - .prepare_for_closing() - .wait(); - - Ok(()) -} - -pub fn load_dump( +pub async fn load_dump( db_path: impl AsRef, dump_path: impl AsRef, size: usize, @@ -212,15 +185,10 @@ pub fn load_dump( // read dump metadata let metadata = DumpMetadata::from_path(&tmp_dir_path)?; - // choose importation function from DumpVersion of metadata - let import_index = match metadata.dump_version { - DumpVersion::V1 => import_index_v1, - }; - // remove indexes which have same `uuid` than indexes to import and create empty indexes - let existing_index_uids = futures::executor::block_on(uuid_resolver.list())?; + let existing_index_uids = uuid_resolver.list().await?; - info!("Deleting indexes provided in the dump..."); + info!("Deleting indexes already present in the db and provided in the dump..."); for idx in &metadata.indexes { if let Some((_, uuid)) = existing_index_uids.iter().find(|(s, _)| s == &idx.uid) { // if we find the index in the `uuid_resolver` it's supposed to exist on the file system @@ -237,18 +205,19 @@ pub fn load_dump( } } else { // if the index does not exist in the `uuid_resolver` we create it - futures::executor::block_on(uuid_resolver.create(idx.uid.clone()))?; + uuid_resolver.create(idx.uid.clone()).await?; } } // import each indexes content for idx in metadata.indexes { let dump_path = tmp_dir_path.join(&idx.uid); - let uuid = futures::executor::block_on(uuid_resolver.get(idx.uid))?; + let uuid = uuid_resolver.get(idx.uid).await?; let index_path = db_path.join(&format!("indexes/index-{}", uuid)); + let update_path = db_path.join(&format!("updates/updates-{}", uuid)); // TODO: add the update db info!("Importing dump from {} into {}...", dump_path.display(), index_path.display()); - import_index(size, &dump_path, &index_path).unwrap(); + metadata.dump_version.import_index(size, &dump_path, &index_path).unwrap(); info!("Dump importation from {} succeed", dump_path.display()); } diff --git a/meilisearch-http/src/index_controller/dump/v1.rs b/meilisearch-http/src/index_controller/dump/v1.rs new file mode 100644 index 000000000..433d529e1 --- /dev/null +++ b/meilisearch-http/src/index_controller/dump/v1.rs @@ -0,0 +1,119 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use log::warn; +use serde::{Deserialize, Serialize}; +use crate::index_controller; +use super::*; + +/// This is the settings used in the last version of meilisearch exporting dump in V1 +#[derive(Default, Clone, Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Settings { + #[serde(default, deserialize_with = "deserialize_some")] + pub ranking_rules: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub distinct_attribute: Option>, + #[serde(default, deserialize_with = "deserialize_some")] + pub searchable_attributes: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub displayed_attributes: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub stop_words: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub synonyms: Option>>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub attributes_for_faceting: Option>>, +} + +/// we need to **always** be able to convert the old settings to the settings currently being used +impl From for index_controller::Settings { + fn from(settings: Settings) -> Self { + if settings.distinct_attribute.flatten().is_some() { + error!("`distinct_attribute` are not yet implemented and thus will be ignored"); + } + if settings.synonyms.flatten().is_some() { + error!("`synonyms` are not yet implemented and thus will be ignored"); + } + Self { + // we need to convert the old `Vec` into a `BTreeSet` + displayed_attributes: settings.displayed_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), + searchable_attributes: settings.searchable_attributes, + // we previously had a `Vec` but now we have a `HashMap` + // representing the name of the faceted field + the type of the field. Since the type + // was not known in the V1 of the dump we are just going to assume everything is a + // String + attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().map(|key| (key, String::from("string"))).collect())), + // we need to convert the old `Vec` into a `BTreeSet` + ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { + match criterion.as_str() { + "words" | "typo" | "proximity" => Some(criterion), + s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), + "wordsPosition" => { + warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); + Some(String::from("words")) + } + "attribute" | "exactness" => { + error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); + None + } + s => { + error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); + None + } + } + }).collect())), + // we need to convert the old `Vec` into a `BTreeSet` + stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())), + } + } +} + +/// Extract Settings from `settings.json` file present at provided `dir_path` +fn import_settings(dir_path: &Path) -> anyhow::Result { + let path = dir_path.join("settings.json"); + let file = File::open(path)?; + let reader = std::io::BufReader::new(file); + let metadata = serde_json::from_reader(reader)?; + + Ok(metadata) +} + + +pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { + info!("Importing a dump from an old version of meilisearch with dump version 1"); + + std::fs::create_dir_all(&index_path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, index_path)?; + let index = Index(Arc::new(index)); + + // extract `settings.json` file and import content + let settings = import_settings(&dump_path)?; + dbg!(&settings); + let settings = settings.into(); + dbg!(&settings); + let update_builder = UpdateBuilder::new(0); + index.update_settings(&settings, update_builder)?; + + let update_builder = UpdateBuilder::new(1); + let file = File::open(&dump_path.join("documents.jsonl"))?; + let reader = std::io::BufReader::new(file); + + index.update_documents( + UpdateFormat::JsonStream, + IndexDocumentsMethod::ReplaceDocuments, + reader, + update_builder, + None, + )?; + + // the last step: we extract the original milli::Index and close it + Arc::try_unwrap(index.0) + .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") + .unwrap() + .prepare_for_closing() + .wait(); + + Ok(()) +} diff --git a/meilisearch-http/src/index_controller/dump/v2.rs b/meilisearch-http/src/index_controller/dump/v2.rs new file mode 100644 index 000000000..f9303af0d --- /dev/null +++ b/meilisearch-http/src/index_controller/dump/v2.rs @@ -0,0 +1,51 @@ +use heed::EnvOpenOptions; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use crate::index::Index; +use crate::index_controller::Settings; +use std::{fs::File, path::Path, sync::Arc}; + +/// Extract Settings from `settings.json` file present at provided `dir_path` +fn import_settings(dir_path: &Path) -> anyhow::Result { + let path = dir_path.join("settings.json"); + let file = File::open(path)?; + let reader = std::io::BufReader::new(file); + let metadata = serde_json::from_reader(reader)?; + + Ok(metadata) +} + +pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(&index_path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, index_path)?; + let index = Index(Arc::new(index)); + + // extract `settings.json` file and import content + let settings = import_settings(&dump_path)?; + let update_builder = UpdateBuilder::new(0); + index.update_settings(&settings, update_builder)?; + dbg!(settings); + + let update_builder = UpdateBuilder::new(1); + let file = File::open(&dump_path.join("documents.jsonl"))?; + let reader = std::io::BufReader::new(file); + + index.update_documents( + UpdateFormat::JsonStream, + IndexDocumentsMethod::ReplaceDocuments, + reader, + update_builder, + None, + )?; + + // the last step: we extract the original milli::Index and close it + Arc::try_unwrap(index.0) + .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") + .unwrap() + .prepare_for_closing() + .wait(); + + Ok(()) +} + diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 10b9142cc..fe894298d 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -75,7 +75,7 @@ pub struct Stats { } impl IndexController { - pub fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { + pub async fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { let index_size = options.max_mdb_size.get_bytes() as usize; let update_store_size = options.max_udb_size.get_bytes() as usize; @@ -92,7 +92,7 @@ impl IndexController { &options.db_path, path, index_size, - ); + ).await?; } diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index b16f3c0e1..592b70d30 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -54,7 +54,7 @@ async fn main() -> Result<(), MainError> { //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; //} - let data = Data::new(opt.clone())?; + let data = Data::new(opt.clone()).await?; //if !opt.no_analytics { //let analytics_data = data.clone(); From 0fee81678e4f61cbe332d8d88beee32d515c7880 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 10 May 2021 20:22:18 +0200 Subject: [PATCH 312/522] [WIP] rebase on main --- meilisearch-http/src/index/mod.rs | 19 ++++++++++++++ meilisearch-http/src/index/updates.rs | 26 +++++++++---------- .../src/index_controller/dump/mod.rs | 8 ------ .../src/index_controller/dump/v1.rs | 7 ++--- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index c897fac3f..048ed56bb 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -9,6 +9,7 @@ use serde_json::{Map, Value}; use crate::helpers::EnvSizer; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use updates::{Facets, Settings, Checked, Unchecked}; +use serde::{de::Deserializer, Deserialize}; mod search; mod updates; @@ -26,6 +27,22 @@ impl Deref for Index { } } +pub fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: Deserialize<'de>, + D: Deserializer<'de>, +{ + Deserialize::deserialize(deserializer).map(Some) +} + +pub fn deserialize_wildcard<'de, D>(deserializer: D) -> Result>>, D::Error> +where + D: Deserializer<'de>, +{ + Ok(> as Deserialize>::deserialize(deserializer)? + .map(|item: Vec| (!item.iter().any(|s| s == "*")).then(|| item))) +} + impl Index { pub fn settings(&self) -> anyhow::Result> { let txn = self.read_txn()?; @@ -88,6 +105,8 @@ impl Index { let mut documents = Vec::new(); + println!("fields to display: {:?}", fields_to_display); + for entry in iter { let (_id, obkv) = entry?; let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 0d76f2ae6..a3012fe9a 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -5,11 +5,17 @@ use std::marker::PhantomData; use flate2::read::GzDecoder; use log::info; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use serde::{de::Deserializer, Deserialize, Serialize}; +use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use serde::{Deserialize, Serialize}; -use super::Index; -use crate::index_controller::UpdateResult; +use super::{deserialize_some, deserialize_wildcard, Index}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum UpdateResult { + DocumentsAddition(DocumentAdditionResult), + DocumentDeletion { deleted: u64 }, + Other, +} #[derive(Clone, Default, Debug)] pub struct Checked; @@ -22,14 +28,14 @@ pub struct Unchecked; pub struct Settings { #[serde( default, - deserialize_with = "deserialize_some", + deserialize_with = "deserialize_wildcard", skip_serializing_if = "Option::is_none" )] pub displayed_attributes: Option>>, #[serde( default, - deserialize_with = "deserialize_some", + deserialize_with = "deserialize_wildcard", skip_serializing_if = "Option::is_none" )] pub searchable_attributes: Option>>, @@ -118,14 +124,6 @@ pub struct Facets { pub min_level_size: Option, } -fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> -where - T: Deserialize<'de>, - D: Deserializer<'de>, -{ - Deserialize::deserialize(deserializer).map(Some) -} - impl Index { pub fn update_documents( &self, diff --git a/meilisearch-http/src/index_controller/dump/mod.rs b/meilisearch-http/src/index_controller/dump/mod.rs index 7278a7b77..6be9c5161 100644 --- a/meilisearch-http/src/index_controller/dump/mod.rs +++ b/meilisearch-http/src/index_controller/dump/mod.rs @@ -19,14 +19,6 @@ use crate::index::Index; use crate::index_controller::uuid_resolver; use crate::{helpers::compression, index::Settings}; -pub (super) fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> -where - T: Deserialize<'de>, - D: Deserializer<'de>, -{ - Deserialize::deserialize(deserializer).map(Some) -} - #[derive(Debug, Serialize, Deserialize, Copy, Clone)] enum DumpVersion { V1, diff --git a/meilisearch-http/src/index_controller/dump/v1.rs b/meilisearch-http/src/index_controller/dump/v1.rs index 433d529e1..3e82b9084 100644 --- a/meilisearch-http/src/index_controller/dump/v1.rs +++ b/meilisearch-http/src/index_controller/dump/v1.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use log::warn; use serde::{Deserialize, Serialize}; use crate::index_controller; +use crate::index::{deserialize_wildcard, deserialize_some}; use super::*; /// This is the settings used in the last version of meilisearch exporting dump in V1 @@ -13,10 +14,10 @@ struct Settings { pub ranking_rules: Option>>, #[serde(default, deserialize_with = "deserialize_some")] pub distinct_attribute: Option>, - #[serde(default, deserialize_with = "deserialize_some")] + #[serde(default, deserialize_with = "deserialize_wildcard")] pub searchable_attributes: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub displayed_attributes: Option>>, + #[serde(default, deserialize_with = "deserialize_wildcard")] + pub displayed_attributes: Option>>, #[serde(default, deserialize_with = "deserialize_some")] pub stop_words: Option>>, #[serde(default, deserialize_with = "deserialize_some")] From 1b5fc61eb666512bbdd9a9b50a1a1507cea48e1f Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 10 May 2021 20:23:12 +0200 Subject: [PATCH 313/522] [WIP] rebase on main --- meilisearch-http/src/index/mod.rs | 9 +- .../src/index_controller/dump/mod.rs | 13 +- .../src/index_controller/dump/v1.rs | 11 +- meilisearch-http/src/index_controller/mod.rs | 2 +- .../src/index_controller/uuid_resolver/mod.rs | 3 +- .../index_controller/uuid_resolver/store.rs | 263 ++++++++++-------- 6 files changed, 164 insertions(+), 137 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 048ed56bb..b0c145001 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; use crate::helpers::EnvSizer; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; -pub use updates::{Facets, Settings, Checked, Unchecked}; +pub use updates::{Facets, Settings, Checked, Unchecked, UpdateResult}; use serde::{de::Deserializer, Deserialize}; mod search; @@ -35,12 +35,13 @@ where Deserialize::deserialize(deserializer).map(Some) } -pub fn deserialize_wildcard<'de, D>(deserializer: D) -> Result>>, D::Error> +pub fn deserialize_wildcard<'de, I, D>(deserializer: D) -> Result>, D::Error> where D: Deserializer<'de>, + I: IntoIterator + Deserialize<'de> + Clone, { - Ok(> as Deserialize>::deserialize(deserializer)? - .map(|item: Vec| (!item.iter().any(|s| s == "*")).then(|| item))) + Ok( as Deserialize>::deserialize(deserializer)? + .map(|item: I| (!item.clone().into_iter().any(|s| s == "*")).then(|| item))) } impl Index { diff --git a/meilisearch-http/src/index_controller/dump/mod.rs b/meilisearch-http/src/index_controller/dump/mod.rs index 6be9c5161..1f42466cb 100644 --- a/meilisearch-http/src/index_controller/dump/mod.rs +++ b/meilisearch-http/src/index_controller/dump/mod.rs @@ -7,7 +7,7 @@ use anyhow::bail; use heed::EnvOpenOptions; use log::{error, info}; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use serde::{de::Deserializer, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use tempfile::TempDir; use tokio::fs; use tokio::task::spawn_blocking; @@ -159,7 +159,7 @@ fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> Ok(()) } -pub async fn load_dump( +pub fn load_dump( db_path: impl AsRef, dump_path: impl AsRef, size: usize, @@ -167,7 +167,7 @@ pub async fn load_dump( info!("Importing dump from {}...", dump_path.as_ref().display()); let db_path = db_path.as_ref(); let dump_path = dump_path.as_ref(); - let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&db_path)?; + let uuid_resolver = uuid_resolver::HeedUuidStore::new(&db_path)?; // extract the dump in a temporary directory let tmp_dir = TempDir::new()?; @@ -178,7 +178,7 @@ pub async fn load_dump( let metadata = DumpMetadata::from_path(&tmp_dir_path)?; // remove indexes which have same `uuid` than indexes to import and create empty indexes - let existing_index_uids = uuid_resolver.list().await?; + let existing_index_uids = uuid_resolver.list()?; info!("Deleting indexes already present in the db and provided in the dump..."); for idx in &metadata.indexes { @@ -197,14 +197,15 @@ pub async fn load_dump( } } else { // if the index does not exist in the `uuid_resolver` we create it - uuid_resolver.create(idx.uid.clone()).await?; + uuid_resolver.create_uuid(idx.uid.clone(), false)?; } } // import each indexes content for idx in metadata.indexes { let dump_path = tmp_dir_path.join(&idx.uid); - let uuid = uuid_resolver.get(idx.uid).await?; + // this cannot fail since we created all the missing uuid in the previous loop + let uuid = uuid_resolver.get_uuid(idx.uid)?.unwrap(); let index_path = db_path.join(&format!("indexes/index-{}", uuid)); let update_path = db_path.join(&format!("updates/updates-{}", uuid)); // TODO: add the update db diff --git a/meilisearch-http/src/index_controller/dump/v1.rs b/meilisearch-http/src/index_controller/dump/v1.rs index 3e82b9084..02d97e8c6 100644 --- a/meilisearch-http/src/index_controller/dump/v1.rs +++ b/meilisearch-http/src/index_controller/dump/v1.rs @@ -17,7 +17,7 @@ struct Settings { #[serde(default, deserialize_with = "deserialize_wildcard")] pub searchable_attributes: Option>>, #[serde(default, deserialize_with = "deserialize_wildcard")] - pub displayed_attributes: Option>>, + pub displayed_attributes: Option>>, #[serde(default, deserialize_with = "deserialize_some")] pub stop_words: Option>>, #[serde(default, deserialize_with = "deserialize_some")] @@ -92,8 +92,13 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: // extract `settings.json` file and import content let settings = import_settings(&dump_path)?; dbg!(&settings); - let settings = settings.into(); - dbg!(&settings); + let mut settings: index_controller::Settings = settings.into(); + if settings.displayed_attributes.as_ref().map_or(false, |o| o.as_ref().map_or(false, |v| v.contains(&String::from("*")))) { + settings.displayed_attributes = None; + } + if settings.searchable_attributes.as_ref().map_or(false, |o| o.as_ref().map_or(false, |v| v.contains(&String::from("*")))) { + settings.searchable_attributes = None; + } let update_builder = UpdateBuilder::new(0); index.update_settings(&settings, update_builder)?; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index fe894298d..7a67265a7 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -92,7 +92,7 @@ impl IndexController { &options.db_path, path, index_size, - ).await?; + )?; } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index a8361095c..b3f70ba2e 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -11,11 +11,12 @@ use uuid::Uuid; use actor::UuidResolverActor; use message::UuidResolveMsg; -use store::{HeedUuidStore, UuidStore}; +use store::UuidStore; #[cfg(test)] use mockall::automock; +pub use store::HeedUuidStore; pub use handle_impl::UuidResolverHandleImpl; const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index df4c3a2fb..1d387ddc3 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -1,6 +1,6 @@ -use std::path::{Path, PathBuf}; use std::collections::HashSet; use std::fs::create_dir_all; +use std::path::{Path, PathBuf}; use heed::{ types::{ByteSlice, Str}, @@ -25,6 +25,7 @@ pub trait UuidStore { async fn get_size(&self) -> Result; } +#[derive(Clone)] pub struct HeedUuidStore { env: Env, db: Database, @@ -40,150 +41,168 @@ impl HeedUuidStore { let db = env.create_database(None)?; Ok(Self { env, db }) } + + pub fn create_uuid(&self, name: String, err: bool) -> Result { + let env = self.env.clone(); + let db = self.db; + let mut txn = env.write_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + if err { + Err(UuidError::NameAlreadyExist) + } else { + let uuid = Uuid::from_slice(uuid)?; + Ok(uuid) + } + } + None => { + let uuid = Uuid::new_v4(); + db.put(&mut txn, &name, uuid.as_bytes())?; + txn.commit()?; + Ok(uuid) + } + } + } + + pub fn get_uuid(&self, name: String) -> Result> { + let env = self.env.clone(); + let db = self.db; + let txn = env.read_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + Ok(Some(uuid)) + } + None => Ok(None), + } + } + + pub fn delete(&self, uid: String) -> Result> { + let env = self.env.clone(); + let db = self.db; + let mut txn = env.write_txn()?; + match db.get(&txn, &uid)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + db.delete(&mut txn, &uid)?; + txn.commit()?; + Ok(Some(uuid)) + } + None => Ok(None), + } + } + + pub fn list(&self) -> Result> { + let env = self.env.clone(); + let db = self.db; + let txn = env.read_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (name, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push((name.to_owned(), uuid)) + } + Ok(entries) + } + + pub fn insert(&self, name: String, uuid: Uuid) -> Result<()> { + let env = self.env.clone(); + let db = self.db; + let mut txn = env.write_txn()?; + db.put(&mut txn, &name, uuid.as_bytes())?; + txn.commit()?; + Ok(()) + } + + // TODO: we should merge this function and the following function for the dump. it's exactly + // the same code + pub fn snapshot(&self, mut path: PathBuf) -> Result> { + let env = self.env.clone(); + let db = self.db; + // Write transaction to acquire a lock on the database. + let txn = env.write_txn()?; + let mut entries = HashSet::new(); + for entry in db.iter(&txn)? { + let (_, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.insert(uuid); + } + + // only perform snapshot if there are indexes + if !entries.is_empty() { + path.push("index_uuids"); + create_dir_all(&path).unwrap(); + path.push("data.mdb"); + env.copy_to_path(path, CompactionOption::Enabled)?; + } + Ok(entries) + } + + pub fn dump(&self, mut path: PathBuf) -> Result> { + let env = self.env.clone(); + let db = self.db; + // Write transaction to acquire a lock on the database. + let txn = env.write_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (_, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push(uuid) + } + + // only perform dump if there are indexes + if !entries.is_empty() { + path.push("index_uuids"); + create_dir_all(&path).unwrap(); + path.push("data.mdb"); + env.copy_to_path(path, CompactionOption::Enabled)?; + } + Ok(entries) + } + + pub fn get_size(&self) -> Result { + Ok(self.env.size()) + } } #[async_trait::async_trait] impl UuidStore for HeedUuidStore { async fn create_uuid(&self, name: String, err: bool) -> Result { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let mut txn = env.write_txn()?; - match db.get(&txn, &name)? { - Some(uuid) => { - if err { - Err(UuidError::NameAlreadyExist) - } else { - let uuid = Uuid::from_slice(uuid)?; - Ok(uuid) - } - } - None => { - let uuid = Uuid::new_v4(); - db.put(&mut txn, &name, uuid.as_bytes())?; - txn.commit()?; - Ok(uuid) - } - } - }) - .await? + let this = self.clone(); + tokio::task::spawn_blocking(move || this.create_uuid(name, err)).await? } async fn get_uuid(&self, name: String) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let txn = env.read_txn()?; - match db.get(&txn, &name)? { - Some(uuid) => { - let uuid = Uuid::from_slice(uuid)?; - Ok(Some(uuid)) - } - None => Ok(None), - } - }) - .await? + let this = self.clone(); + tokio::task::spawn_blocking(move || this.get_uuid(name)).await? } async fn delete(&self, uid: String) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let mut txn = env.write_txn()?; - match db.get(&txn, &uid)? { - Some(uuid) => { - let uuid = Uuid::from_slice(uuid)?; - db.delete(&mut txn, &uid)?; - txn.commit()?; - Ok(Some(uuid)) - } - None => Ok(None), - } - }) - .await? + let this = self.clone(); + tokio::task::spawn_blocking(move || this.delete(uid)).await? } async fn list(&self) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let txn = env.read_txn()?; - let mut entries = Vec::new(); - for entry in db.iter(&txn)? { - let (name, uuid) = entry?; - let uuid = Uuid::from_slice(uuid)?; - entries.push((name.to_owned(), uuid)) - } - Ok(entries) - }) - .await? + let this = self.clone(); + tokio::task::spawn_blocking(move || this.list()).await? } async fn insert(&self, name: String, uuid: Uuid) -> Result<()> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - let mut txn = env.write_txn()?; - db.put(&mut txn, &name, uuid.as_bytes())?; - txn.commit()?; - Ok(()) - }) - .await? + let this = self.clone(); + tokio::task::spawn_blocking(move || this.insert(name, uuid)).await? } - // TODO: we should merge this function and the following function for the dump. it's exactly - // the same code - async fn snapshot(&self, mut path: PathBuf) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - // Write transaction to acquire a lock on the database. - let txn = env.write_txn()?; - let mut entries = HashSet::new(); - for entry in db.iter(&txn)? { - let (_, uuid) = entry?; - let uuid = Uuid::from_slice(uuid)?; - entries.insert(uuid); - } - - // only perform snapshot if there are indexes - if !entries.is_empty() { - path.push("index_uuids"); - create_dir_all(&path).unwrap(); - path.push("data.mdb"); - env.copy_to_path(path, CompactionOption::Enabled)?; - } - Ok(entries) - }) - .await? + async fn snapshot(&self, path: PathBuf) -> Result> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.snapshot(path)).await? } - async fn dump(&self, mut path: PathBuf) -> Result> { - let env = self.env.clone(); - let db = self.db; - tokio::task::spawn_blocking(move || { - // Write transaction to acquire a lock on the database. - let txn = env.write_txn()?; - let mut entries = Vec::new(); - for entry in db.iter(&txn)? { - let (_, uuid) = entry?; - let uuid = Uuid::from_slice(uuid)?; - entries.push(uuid) - } - - // only perform dump if there are indexes - if !entries.is_empty() { - path.push("index_uuids"); - create_dir_all(&path).unwrap(); - path.push("data.mdb"); - env.copy_to_path(path, CompactionOption::Enabled)?; - } - Ok(entries) - }) - .await? + async fn dump(&self, path: PathBuf) -> Result> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.dump(path)).await? } async fn get_size(&self) -> Result { - Ok(self.env.size()) + self.get_size() } } From 0275b36fb0db758a6fb32b3a933ba2595e9a578d Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 10 May 2021 20:24:14 +0200 Subject: [PATCH 314/522] [WIP] rebase on main --- meilisearch-http/src/index/updates.rs | 9 +- .../src/index_controller/dump/mod.rs | 4 +- .../src/index_controller/dump/v1.rs | 6 +- .../src/index_controller/dump/v2.rs | 2 +- .../src/index_controller/index_actor/actor.rs | 4 +- .../index_actor/handle_impl.rs | 6 +- .../index_controller/index_actor/message.rs | 2 +- .../src/index_controller/index_actor/mod.rs | 2 + meilisearch-http/src/index_controller/mod.rs | 3 + .../index_controller/update_actor/actor.rs | 125 +++++------------- .../update_actor/handle_impl.rs | 11 +- .../index_controller/update_actor/message.rs | 6 +- .../src/index_controller/update_actor/mod.rs | 6 +- .../update_actor/update_store.rs | 55 +++----- .../index_controller/uuid_resolver/actor.rs | 2 +- .../uuid_resolver/handle_impl.rs | 2 +- .../index_controller/uuid_resolver/message.rs | 2 +- .../src/index_controller/uuid_resolver/mod.rs | 2 +- .../index_controller/uuid_resolver/store.rs | 14 +- meilisearch-http/src/routes/index.rs | 4 +- 20 files changed, 93 insertions(+), 174 deletions(-) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index a3012fe9a..75d0dc3e6 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -5,17 +5,12 @@ use std::marker::PhantomData; use flate2::read::GzDecoder; use log::info; -use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; use super::{deserialize_some, deserialize_wildcard, Index}; +use crate::index_controller::UpdateResult; -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateResult { - DocumentsAddition(DocumentAdditionResult), - DocumentDeletion { deleted: u64 }, - Other, -} #[derive(Clone, Default, Debug)] pub struct Checked; diff --git a/meilisearch-http/src/index_controller/dump/mod.rs b/meilisearch-http/src/index_controller/dump/mod.rs index 1f42466cb..c12041e9d 100644 --- a/meilisearch-http/src/index_controller/dump/mod.rs +++ b/meilisearch-http/src/index_controller/dump/mod.rs @@ -9,8 +9,6 @@ use log::{error, info}; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; use tempfile::TempDir; -use tokio::fs; -use tokio::task::spawn_blocking; use super::update_actor::UpdateActorHandle; use super::uuid_resolver::UuidResolverHandle; @@ -109,6 +107,7 @@ where } async fn perform_dump(&self) -> anyhow::Result<()> { + /* info!("Performing dump."); let dump_dir = self.dump_path.clone(); @@ -144,6 +143,7 @@ where .await??; info!("Created dump in {:?}.", dump_path); + */ Ok(()) } diff --git a/meilisearch-http/src/index_controller/dump/v1.rs b/meilisearch-http/src/index_controller/dump/v1.rs index 02d97e8c6..89b55c51e 100644 --- a/meilisearch-http/src/index_controller/dump/v1.rs +++ b/meilisearch-http/src/index_controller/dump/v1.rs @@ -29,13 +29,11 @@ struct Settings { /// we need to **always** be able to convert the old settings to the settings currently being used impl From for index_controller::Settings { fn from(settings: Settings) -> Self { - if settings.distinct_attribute.flatten().is_some() { - error!("`distinct_attribute` are not yet implemented and thus will be ignored"); - } if settings.synonyms.flatten().is_some() { error!("`synonyms` are not yet implemented and thus will be ignored"); } Self { + distinct_attribute: settings.distinct_attribute, // we need to convert the old `Vec` into a `BTreeSet` displayed_attributes: settings.displayed_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), searchable_attributes: settings.searchable_attributes, @@ -109,7 +107,7 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: index.update_documents( UpdateFormat::JsonStream, IndexDocumentsMethod::ReplaceDocuments, - reader, + Some(reader), update_builder, None, )?; diff --git a/meilisearch-http/src/index_controller/dump/v2.rs b/meilisearch-http/src/index_controller/dump/v2.rs index f9303af0d..7b9a56772 100644 --- a/meilisearch-http/src/index_controller/dump/v2.rs +++ b/meilisearch-http/src/index_controller/dump/v2.rs @@ -34,7 +34,7 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: index.update_documents( UpdateFormat::JsonStream, IndexDocumentsMethod::ReplaceDocuments, - reader, + Some(reader), update_builder, None, )?; diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 535c405dc..ecf71b62b 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -312,7 +312,7 @@ impl IndexActor { Ok(()) } - async fn handle_dump(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> { + async fn handle_dump(&self, uuid: Uuid, mut path: PathBuf) -> IndexResult<()> { use tokio::fs::create_dir_all; path.push("indexes"); @@ -340,7 +340,7 @@ impl IndexActor { Ok(()) } - async fn handle_get_stats(&self, uuid: Uuid) -> Result { + async fn handle_get_stats(&self, uuid: Uuid) -> IndexResult { let index = self .store .get(uuid) diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index d625a763e..26aa189d0 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -136,14 +136,14 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Dump { uuid, path, ret }; - let _ = self.read_sender.send(msg).await; + let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn get_index_stats(&self, uuid: Uuid) -> Result { + async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetStats { uuid, ret }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 0d88532ca..714a30ecc 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -63,7 +63,7 @@ pub enum IndexMsg { Dump { uuid: Uuid, path: PathBuf, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, GetStats { uuid: Uuid, diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 46105742b..7522acc67 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -97,6 +97,7 @@ pub trait IndexActorHandle { index_settings: IndexSettings, ) -> IndexResult; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; + async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; async fn get_index_stats(&self, uuid: Uuid) -> IndexResult; } @@ -180,4 +181,5 @@ mod test { async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { self.as_ref().get_index_stats(uuid).await } + } } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 7a67265a7..9ed6f10e4 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -5,6 +5,7 @@ use std::time::Duration; use actix_web::web::{Bytes, Payload}; use anyhow::bail; +use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use log::info; use milli::FieldsDistribution; @@ -22,6 +23,8 @@ use uuid_resolver::{UuidError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; +use self::dump::load_dump; + mod index_actor; mod snapshot; mod dump; diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 7885d0b3b..6789bc6ce 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -13,7 +13,7 @@ use tokio::sync::mpsc; use uuid::Uuid; use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore, UpdateStoreInfo}; -use crate::index_controller::index_actor::{IndexActorHandle, CONCURRENT_INDEX_MSG}; +use crate::index_controller::{index_actor::{IndexActorHandle, CONCURRENT_INDEX_MSG}}; use crate::index_controller::{UpdateMeta, UpdateStatus}; pub struct UpdateActor { @@ -71,16 +71,14 @@ where Some(Delete { uuid, ret }) => { let _ = ret.send(self.handle_delete(uuid).await); } - Some(Snapshot { uuid, path, ret }) => { - let _ = ret.send(self.handle_snapshot(uuid, path).await); + Some(Snapshot { uuids, path, ret }) => { + let _ = ret.send(self.handle_snapshot(uuids, path).await); } - Some(Dump { uuid, path, ret }) => { - let _ = ret.send(self.handle_dump(uuid, path).await); + Some(Dump { uuids, path, ret }) => { + let _ = ret.send(self.handle_dump(uuids, path).await); } Some(GetInfo { ret }) => { let _ = ret.send(self.handle_get_info().await); - Some(GetSize { uuid, ret }) => { - let _ = ret.send(self.handle_get_size(uuid).await); } None => break, } @@ -199,51 +197,9 @@ where } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let store = self.store.delete(uuid).await?; + let store = self.store.clone(); - if let Some(store) = store { - tokio::task::spawn(async move { - let store = get_arc_ownership_blocking(store).await; - tokio::task::spawn_blocking(move || { - store.prepare_for_closing().wait(); - info!("Update store {} was closed.", uuid); - }); - }); - } - - Ok(()) - } - - async fn handle_create(&self, uuid: Uuid) -> Result<()> { - let _ = self.store.get_or_create(uuid).await?; - Ok(()) - } - - Ok(()) - } - - async fn handle_create(&self, uuid: Uuid) -> Result<()> { - let _ = self.store.get_or_create(uuid).await?; - Ok(()) - } - - async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { - let index_handle = self.index_handle.clone(); - if let Some(update_store) = self.store.get(uuid).await? { - tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - // acquire write lock to prevent further writes during snapshot - // the update lock must be acquired BEFORE the write lock to prevent dead lock - let _lock = update_store.update_lock.lock(); - let mut txn = update_store.env.write_txn()?; - - // create db snapshot - update_store.snapshot(&mut txn, &path, uuid)?; - - futures::executor::block_on( - async move { index_handle.snapshot(uuid, path).await }, - )?; - Ok(()) - }) + tokio::task::spawn_blocking(move || store.delete_all(uuid)) .await .map_err(|e| UpdateError::Error(e.into()))? .map_err(|e| UpdateError::Error(e.into()))?; @@ -280,6 +236,35 @@ where Ok(()) } + async fn handle_dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + let index_handle = self.index_handle.clone(); + let update_store = self.store.clone(); + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + update_store.dump(&uuids, &path)?; + + // Perform the snapshot of each index concurently. Only a third of the capabilities of + // the index actor at a time not to put too much pressure on the index actor + let path = &path; + let handle = &index_handle; + + let mut stream = futures::stream::iter(uuids.iter()) + .map(|&uuid| handle.dump(uuid, path.clone())) + .buffer_unordered(CONCURRENT_INDEX_MSG / 3); + + Handle::current().block_on(async { + while let Some(res) = stream.next().await { + res?; + } + Ok(()) + }) + }) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?; + + Ok(()) + } + async fn handle_get_info(&self) -> Result { let update_store = self.store.clone(); let info = tokio::task::spawn_blocking(move || -> anyhow::Result { @@ -292,42 +277,4 @@ where Ok(info) } - - async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { - let index_handle = self.index_handle.clone(); - if let Some(update_store) = self.store.get(uuid).await? { - tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - // acquire write lock to prevent further writes during the dump - // the update lock must be acquired BEFORE the write lock to prevent dead lock - let _lock = update_store.update_lock.lock(); - let mut txn = update_store.env.write_txn()?; - - // create db dump - update_store.dump(&mut txn, &path, uuid)?; - - futures::executor::block_on( - async move { index_handle.dump(uuid, path).await }, - )?; - Ok(()) - }) - .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?; - } - - Ok(()) - } - - async fn handle_get_size(&self, uuid: Uuid) -> Result { - let size = match self.store.get(uuid).await? { - Some(update_store) => tokio::task::spawn_blocking(move || -> anyhow::Result { - let txn = update_store.env.read_txn()?; - - update_store.get_size(&txn) - }) - .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?, - None => 0, - }; } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 569b896b0..09a242377 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -78,16 +78,9 @@ where receiver.await.expect("update actor killed.") } - async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Dump { uuid, path, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") - } - - async fn get_size(&self, uuid: Uuid) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::GetSize { uuid, ret }; + let msg = UpdateMsg::Dump { uuids, path, ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 3f39c224f..37df2af32 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -32,15 +32,11 @@ pub enum UpdateMsg { ret: oneshot::Sender>, }, Dump { - uuid: Uuid, + uuids: HashSet, path: PathBuf, ret: oneshot::Sender>, }, GetInfo { ret: oneshot::Sender>, }, - GetSize { - uuid: Uuid, - ret: oneshot::Sender>, - }, } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 4d8ab6f20..36390c290 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -40,11 +40,9 @@ pub trait UpdateActorHandle { async fn get_all_updates_status(&self, uuid: Uuid) -> Result>; async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; - async fn create(&self, uuid: Uuid) -> Result<()>; - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; - async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn snapshot(&self, uuid: HashSet, path: PathBuf) -> Result<()>; + async fn dump(&self, uuid: HashSet, path: PathBuf) -> Result<()>; async fn get_info(&self) -> Result; - async fn get_size(&self, uuid: Uuid) -> Result; async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 4bc4c8c75..6f698e693 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -499,31 +499,37 @@ impl UpdateStore { Ok(()) } - pub fn dump( - &self, - txn: &mut heed::RwTxn, - path: impl AsRef, - uuid: Uuid, - ) -> anyhow::Result<()> { + pub fn dump(&self, uuids: &HashSet, path: impl AsRef) -> anyhow::Result<()> { + let state_lock = self.state.write(); + state_lock.swap(State::Snapshoting); // TODO: rename the state + + let txn = self.env.write_txn()?; + let update_path = path.as_ref().join("updates"); create_dir_all(&update_path)?; - let mut dump_path = update_path.join(format!("update-{}", uuid)); // acquire write lock to prevent further writes during dump - create_dir_all(&dump_path)?; - dump_path.push("data.mdb"); + create_dir_all(&update_path)?; + let db_path = update_path.join("data.mdb"); + // TODO: everything // create db dump - self.env.copy_to_path(&dump_path, CompactionOption::Enabled)?; + self.env.copy_to_path(&db_path, CompactionOption::Enabled)?; let update_files_path = update_path.join("update_files"); create_dir_all(&update_files_path)?; - for path in self.pending.iter(&txn)? { - let (_, path) = path?; - let name = path.file_name().unwrap(); - let to = update_files_path.join(name); - copy(path, to)?; + let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); + + for entry in pendings { + let ((_, uuid, _), pending) = entry?; + if uuids.contains(&uuid) { + if let Some(path) = pending.decode()?.content_path() { + let name = path.file_name().unwrap(); + let to = update_files_path.join(name); + copy(path, to)?; + } + } } Ok(()) @@ -545,25 +551,6 @@ impl UpdateStore { Ok(UpdateStoreInfo { size, processing }) } - - pub fn get_size(&self, txn: &heed::RoTxn) -> anyhow::Result { - let mut size = self.env.size(); - let txn = self.env.read_txn()?; - - for entry in self.pending_queue.iter(&txn)? { - let (_, pending) = entry?; - if let Some(path) = pending.content_path() { - size += File::open(path)?.metadata()?.len(); - } - } - - let processing = match *self.state.read() { - State::Processing(uuid, _) => Some(uuid), - _ => None, - }; - - Ok(UpdateStoreInfo { size, processing }) - } } #[cfg(test)] diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 9c180e4a8..df83ceba9 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -85,7 +85,7 @@ impl UuidResolverActor { self.store.snapshot(path).await } - async fn handle_dump(&self, path: PathBuf) -> Result> { + async fn handle_dump(&self, path: PathBuf) -> Result> { self.store.dump(path).await } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index e47f9a8e0..d9e9a20fc 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -78,7 +78,7 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - async fn dump(&self, path: PathBuf) -> Result> { + async fn dump(&self, path: PathBuf) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::DumpRequest { path, ret }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index 67493c2cd..78f62eea2 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -33,7 +33,7 @@ pub enum UuidResolveMsg { }, DumpRequest { path: PathBuf, - ret: oneshot::Sender>>, + ret: oneshot::Sender>>, }, GetSize { ret: oneshot::Sender>, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index b3f70ba2e..aca730db9 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -32,7 +32,7 @@ pub trait UuidResolverHandle { async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; async fn snapshot(&self, path: PathBuf) -> Result>; - async fn dump(&self, path: PathBuf) -> Result>; + async fn dump(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 1d387ddc3..917e0b4a5 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -21,7 +21,7 @@ pub trait UuidStore { async fn list(&self) -> Result>; async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; async fn snapshot(&self, path: PathBuf) -> Result>; - async fn dump(&self, path: PathBuf) -> Result>; + async fn dump(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } @@ -116,7 +116,7 @@ impl HeedUuidStore { // TODO: we should merge this function and the following function for the dump. it's exactly // the same code - pub fn snapshot(&self, mut path: PathBuf) -> Result> { + pub fn snapshot(&self, mut path: PathBuf) -> Result> { let env = self.env.clone(); let db = self.db; // Write transaction to acquire a lock on the database. @@ -138,16 +138,16 @@ impl HeedUuidStore { Ok(entries) } - pub fn dump(&self, mut path: PathBuf) -> Result> { + pub fn dump(&self, mut path: PathBuf) -> Result> { let env = self.env.clone(); let db = self.db; // Write transaction to acquire a lock on the database. let txn = env.write_txn()?; - let mut entries = Vec::new(); + let mut entries = HashSet::new(); for entry in db.iter(&txn)? { let (_, uuid) = entry?; let uuid = Uuid::from_slice(uuid)?; - entries.push(uuid) + entries.insert(uuid); } // only perform dump if there are indexes @@ -192,12 +192,12 @@ impl UuidStore for HeedUuidStore { tokio::task::spawn_blocking(move || this.insert(name, uuid)).await? } - async fn snapshot(&self, path: PathBuf) -> Result> { + async fn snapshot(&self, path: PathBuf) -> Result> { let this = self.clone(); tokio::task::spawn_blocking(move || this.snapshot(path)).await? } - async fn dump(&self, path: PathBuf) -> Result> { + async fn dump(&self, path: PathBuf) -> Result> { let this = self.clone(); tokio::task::spawn_blocking(move || this.dump(path)).await? } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 1afc01806..62717c90d 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -1,7 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; -use chrono::DateTime; -use serde::Deserialize; +use chrono::{DateTime, Utc}; +use serde::{Serialize, Deserialize}; use crate::error::ResponseError; use crate::helpers::Authentication; From 0f94ef8abc5fa3e994afb19bccab8f5f7f571757 Mon Sep 17 00:00:00 2001 From: tamo Date: Thu, 29 Apr 2021 14:45:08 +0200 Subject: [PATCH 315/522] WIP: dump --- meilisearch-http/src/data/mod.rs | 4 +- .../src/index_controller/dump/mod.rs | 34 ++++++++++------- .../src/index_controller/dump/v1.rs | 13 +++---- .../src/index_controller/index_actor/mod.rs | 4 ++ meilisearch-http/src/index_controller/mod.rs | 2 +- .../index_controller/update_actor/actor.rs | 1 - .../src/index_controller/update_actor/mod.rs | 3 +- .../update_actor/update_store.rs | 37 ++++++++++++++----- meilisearch-http/src/main.rs | 2 +- .../tests/settings/get_settings.rs | 4 +- 10 files changed, 64 insertions(+), 40 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index e2bb7fbfb..c7979210e 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -55,10 +55,10 @@ impl ApiKeys { } impl Data { - pub async fn new(options: Opt) -> anyhow::Result { + pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - let index_controller = IndexController::new(&path, &options).await?; + let index_controller = IndexController::new(&path, &options)?; let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/meilisearch-http/src/index_controller/dump/mod.rs b/meilisearch-http/src/index_controller/dump/mod.rs index c12041e9d..2bd5f167e 100644 --- a/meilisearch-http/src/index_controller/dump/mod.rs +++ b/meilisearch-http/src/index_controller/dump/mod.rs @@ -1,7 +1,7 @@ mod v1; mod v2; -use std::{fs::File, path::{Path, PathBuf}, sync::Arc}; +use std::{fs::File, path::{Path}, sync::Arc}; use anyhow::bail; use heed::EnvOpenOptions; @@ -10,12 +10,10 @@ use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; use tempfile::TempDir; -use super::update_actor::UpdateActorHandle; -use super::uuid_resolver::UuidResolverHandle; use super::IndexMetadata; use crate::index::Index; use crate::index_controller::uuid_resolver; -use crate::{helpers::compression, index::Settings}; +use crate::helpers::compression; #[derive(Debug, Serialize, Deserialize, Copy, Clone)] enum DumpVersion { @@ -24,7 +22,7 @@ enum DumpVersion { } impl DumpVersion { - const CURRENT: Self = Self::V2; + // const CURRENT: Self = Self::V2; /// Select the good importation function from the `DumpVersion` of metadata pub fn import_index(self, size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { @@ -37,23 +35,25 @@ impl DumpVersion { #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct DumpMetadata { +pub struct Metadata { indexes: Vec, db_version: String, dump_version: DumpVersion, } -impl DumpMetadata { - /// Create a DumpMetadata with the current dump version of meilisearch. +impl Metadata { + /* + /// Create a Metadata with the current dump version of meilisearch. pub fn new(indexes: Vec, db_version: String) -> Self { - DumpMetadata { + Metadata { indexes, db_version, dump_version: DumpVersion::CURRENT, } } + */ - /// Extract DumpMetadata from `metadata.json` file present at provided `dir_path` + /// Extract Metadata from `metadata.json` file present at provided `dir_path` fn from_path(dir_path: &Path) -> anyhow::Result { let path = dir_path.join("metadata.json"); let file = File::open(path)?; @@ -63,7 +63,8 @@ impl DumpMetadata { Ok(metadata) } - /// Write DumpMetadata in `metadata.json` file at provided `dir_path` + /* + /// Write Metadata in `metadata.json` file at provided `dir_path` fn to_path(&self, dir_path: &Path) -> anyhow::Result<()> { let path = dir_path.join("metadata.json"); let file = File::create(path)?; @@ -72,8 +73,10 @@ impl DumpMetadata { Ok(()) } + */ } +/* pub struct DumpService { uuid_resolver_handle: R, update_handle: U, @@ -148,7 +151,9 @@ where Ok(()) } } +*/ +/* /// Write Settings in `settings.json` file at provided `dir_path` fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> { let path = dir_path.join("settings.json"); @@ -158,6 +163,7 @@ fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> Ok(()) } +*/ pub fn load_dump( db_path: impl AsRef, @@ -170,12 +176,12 @@ pub fn load_dump( let uuid_resolver = uuid_resolver::HeedUuidStore::new(&db_path)?; // extract the dump in a temporary directory - let tmp_dir = TempDir::new()?; + let tmp_dir = TempDir::new_in(db_path)?; let tmp_dir_path = tmp_dir.path(); compression::from_tar_gz(dump_path, tmp_dir_path)?; // read dump metadata - let metadata = DumpMetadata::from_path(&tmp_dir_path)?; + let metadata = Metadata::from_path(&tmp_dir_path)?; // remove indexes which have same `uuid` than indexes to import and create empty indexes let existing_index_uids = uuid_resolver.list()?; @@ -207,7 +213,7 @@ pub fn load_dump( // this cannot fail since we created all the missing uuid in the previous loop let uuid = uuid_resolver.get_uuid(idx.uid)?.unwrap(); let index_path = db_path.join(&format!("indexes/index-{}", uuid)); - let update_path = db_path.join(&format!("updates/updates-{}", uuid)); // TODO: add the update db + // let update_path = db_path.join(&format!("updates/updates-{}", uuid)); // TODO: add the update db info!("Importing dump from {} into {}...", dump_path.display(), index_path.display()); metadata.dump_version.import_index(size, &dump_path, &index_path).unwrap(); diff --git a/meilisearch-http/src/index_controller/dump/v1.rs b/meilisearch-http/src/index_controller/dump/v1.rs index 89b55c51e..3a20299f3 100644 --- a/meilisearch-http/src/index_controller/dump/v1.rs +++ b/meilisearch-http/src/index_controller/dump/v1.rs @@ -84,19 +84,13 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: std::fs::create_dir_all(&index_path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options, index_path)?; + let index = milli::Index::new(options.clone(), index_path)?; let index = Index(Arc::new(index)); // extract `settings.json` file and import content let settings = import_settings(&dump_path)?; dbg!(&settings); - let mut settings: index_controller::Settings = settings.into(); - if settings.displayed_attributes.as_ref().map_or(false, |o| o.as_ref().map_or(false, |v| v.contains(&String::from("*")))) { - settings.displayed_attributes = None; - } - if settings.searchable_attributes.as_ref().map_or(false, |o| o.as_ref().map_or(false, |v| v.contains(&String::from("*")))) { - settings.searchable_attributes = None; - } + let settings: index_controller::Settings = settings.into(); let update_builder = UpdateBuilder::new(0); index.update_settings(&settings, update_builder)?; @@ -112,6 +106,9 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: None, )?; + // at this point we should handle the updates, but since the update logic is not handled in + // meilisearch we are just going to ignore this part + // the last step: we extract the original milli::Index and close it Arc::try_unwrap(index.0) .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 7522acc67..e06658ff8 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -178,6 +178,10 @@ mod test { self.as_ref().snapshot(uuid, path).await } + async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + self.as_ref().dump(uuid, path).await + } + async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { self.as_ref().get_index_stats(uuid).await } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 9ed6f10e4..ebcc9ed76 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -78,7 +78,7 @@ pub struct Stats { } impl IndexController { - pub async fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { + pub fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { let index_size = options.max_mdb_size.get_bytes() as usize; let update_store_size = options.max_udb_size.get_bytes() as usize; diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 6789bc6ce..fee907001 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -91,7 +91,6 @@ where meta: UpdateMeta, mut payload: mpsc::Receiver>, ) -> Result { - let file_path = match meta { UpdateMeta::DocumentsAddition { .. } | UpdateMeta::DeleteDocuments => { diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 36390c290..eeca6629f 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -13,9 +13,8 @@ use crate::index_controller::{UpdateMeta, UpdateStatus}; use actor::UpdateActor; use message::UpdateMsg; -use update_store::UpdateStore; -pub use update_store::UpdateStoreInfo; +pub use update_store::{UpdateStore, UpdateStoreInfo}; pub use handle_impl::UpdateActorHandleImpl; pub type Result = std::result::Result; diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 6f698e693..27018764f 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -177,11 +177,7 @@ pub struct UpdateStore { } impl UpdateStore { - pub fn open( - mut options: EnvOpenOptions, - path: impl AsRef, - index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static, - ) -> anyhow::Result> { + pub fn create(mut options: EnvOpenOptions, path: impl AsRef) -> anyhow::Result<(Self, mpsc::Receiver<()>)> { options.max_dbs(5); let env = options.open(path)?; @@ -189,21 +185,30 @@ impl UpdateStore { let next_update_id = env.create_database(Some("next-update-id"))?; let updates = env.create_database(Some("updates"))?; - let (notification_sender, mut notification_receiver) = mpsc::channel(10); + let state = Arc::new(StateLock::from_state(State::Idle)); + + let (notification_sender, notification_receiver) = mpsc::channel(10); // Send a first notification to trigger the process. let _ = notification_sender.send(()); - let state = Arc::new(StateLock::from_state(State::Idle)); + Ok((Self { env, pending_queue, next_update_id, updates, state, notification_sender }, notification_receiver)) + } + + pub fn open( + options: EnvOpenOptions, + path: impl AsRef, + index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static, + ) -> anyhow::Result> { + let (update_store, mut notification_receiver) = Self::create(options, path)?; + let update_store = Arc::new(update_store); // Init update loop to perform any pending updates at launch. // Since we just launched the update store, and we still own the receiving end of the // channel, this call is guaranteed to succeed. - notification_sender + update_store.notification_sender .try_send(()) .expect("Failed to init update store"); - let update_store = Arc::new(UpdateStore { env, pending_queue, next_update_id, updates, state, notification_sender }); - // We need a weak reference so we can take ownership on the arc later when we // want to close the index. let update_store_weak = Arc::downgrade(&update_store); @@ -283,6 +288,18 @@ impl UpdateStore { Ok(meta) } + /// Push already processed updates in the UpdateStore. This is useful for the dumps + pub fn register_already_processed_update ( + &self, + result: UpdateStatus, + index_uuid: Uuid, + ) -> heed::Result<()> { + let mut wtxn = self.env.write_txn()?; + let (_global_id, update_id) = self.next_update_id(&mut wtxn, index_uuid)?; + self.updates.remap_key_type::().put(&mut wtxn, &(index_uuid, update_id), &result)?; + wtxn.commit() + } + /// Executes the user provided function on the next pending update (the one with the lowest id). /// This is asynchronous as it let the user process the update with a read-only txn and /// only writing the result meta to the processed-meta store *after* it has been processed. diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 592b70d30..b16f3c0e1 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -54,7 +54,7 @@ async fn main() -> Result<(), MainError> { //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; //} - let data = Data::new(opt.clone()).await?; + let data = Data::new(opt.clone())?; //if !opt.no_analytics { //let analytics_data = data.clone(); diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index e5f51d7f0..a39dd54e9 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -19,7 +19,7 @@ async fn get_settings() { assert_eq!(settings.keys().len(), 6); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); - assert_eq!(settings["attributesForFaceting"], json!(null)); + assert_eq!(settings["attributesForFaceting"], json!({})); assert_eq!(settings["distinctAttribute"], json!(null)); assert_eq!( settings["rankingRules"], @@ -82,7 +82,9 @@ async fn reset_all_settings() { assert_eq!(response["searchableAttributes"], json!(["bar"])); assert_eq!(response["stopWords"], json!(["the"])); + eprintln!("BEFORE"); index.delete_settings().await; + eprintln!("AFTER"); index.wait_update_id(1).await; let (response, code) = index.settings().await; From c3552cecdfb4f80d2a36279797d49c151725a59e Mon Sep 17 00:00:00 2001 From: tamo Date: Wed, 5 May 2021 14:11:56 +0200 Subject: [PATCH 316/522] WIP rebase on main --- meilisearch-http/src/data/mod.rs | 4 + .../src/index_controller/dump/mod.rs | 154 +++++++----------- .../src/index_controller/index_actor/actor.rs | 50 ++++-- .../index_actor/handle_impl.rs | 4 +- .../index_controller/index_actor/message.rs | 1 + .../src/index_controller/index_actor/mod.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 9 +- .../index_controller/update_actor/actor.rs | 6 +- .../update_actor/handle_impl.rs | 8 +- .../index_controller/update_actor/message.rs | 2 +- .../src/index_controller/update_actor/mod.rs | 2 +- .../update_actor/update_store.rs | 30 ++-- .../index_controller/uuid_resolver/actor.rs | 7 - .../uuid_resolver/handle_impl.rs | 10 -- .../index_controller/uuid_resolver/message.rs | 4 - .../src/index_controller/uuid_resolver/mod.rs | 1 - .../index_controller/uuid_resolver/store.rs | 30 ---- meilisearch-http/src/lib.rs | 4 +- meilisearch-http/src/routes/dump.rs | 25 ++- meilisearch-http/src/routes/mod.rs | 2 +- 20 files changed, 158 insertions(+), 197 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index c7979210e..39cfed626 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -108,6 +108,10 @@ impl Data { Ok(self.index_controller.get_all_stats().await?) } + pub async fn dump(&self) -> anyhow::Result { + Ok(self.index_controller.dump(self.options.dumps_dir.clone()).await?) + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/meilisearch-http/src/index_controller/dump/mod.rs b/meilisearch-http/src/index_controller/dump/mod.rs index 2bd5f167e..a44d4235b 100644 --- a/meilisearch-http/src/index_controller/dump/mod.rs +++ b/meilisearch-http/src/index_controller/dump/mod.rs @@ -1,16 +1,20 @@ mod v1; mod v2; -use std::{fs::File, path::{Path}, sync::Arc}; +use std::{collections::HashSet, fs::{File}, path::{Path, PathBuf}, sync::Arc}; use anyhow::bail; +use chrono::Utc; use heed::EnvOpenOptions; use log::{error, info}; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; use tempfile::TempDir; +use tokio::task::spawn_blocking; +use tokio::fs; +use uuid::Uuid; -use super::IndexMetadata; +use super::{IndexController, IndexMetadata, update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle}; use crate::index::Index; use crate::index_controller::uuid_resolver; use crate::helpers::compression; @@ -22,7 +26,7 @@ enum DumpVersion { } impl DumpVersion { - // const CURRENT: Self = Self::V2; + const CURRENT: Self = Self::V2; /// Select the good importation function from the `DumpVersion` of metadata pub fn import_index(self, size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { @@ -42,7 +46,6 @@ pub struct Metadata { } impl Metadata { - /* /// Create a Metadata with the current dump version of meilisearch. pub fn new(indexes: Vec, db_version: String) -> Self { Metadata { @@ -51,7 +54,6 @@ impl Metadata { dump_version: DumpVersion::CURRENT, } } - */ /// Extract Metadata from `metadata.json` file present at provided `dir_path` fn from_path(dir_path: &Path) -> anyhow::Result { @@ -63,105 +65,73 @@ impl Metadata { Ok(metadata) } - /* /// Write Metadata in `metadata.json` file at provided `dir_path` - fn to_path(&self, dir_path: &Path) -> anyhow::Result<()> { + pub async fn to_path(&self, dir_path: &Path) -> anyhow::Result<()> { let path = dir_path.join("metadata.json"); - let file = File::create(path)?; - - serde_json::to_writer(file, &self)?; - - Ok(()) - } - */ -} - -/* -pub struct DumpService { - uuid_resolver_handle: R, - update_handle: U, - dump_path: PathBuf, - db_name: String, -} - -impl DumpService -where - U: UpdateActorHandle, - R: UuidResolverHandle, -{ - pub fn new( - uuid_resolver_handle: R, - update_handle: U, - dump_path: PathBuf, - db_name: String, - ) -> Self { - Self { - uuid_resolver_handle, - update_handle, - dump_path, - db_name, - } - } - - pub async fn run(self) { - if let Err(e) = self.perform_dump().await { - error!("{}", e); - } - } - - async fn perform_dump(&self) -> anyhow::Result<()> { - /* - info!("Performing dump."); - - let dump_dir = self.dump_path.clone(); - fs::create_dir_all(&dump_dir).await?; - let temp_dump_dir = spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); - - let uuids = self - .uuid_resolver_handle - .dump(temp_dump_path.clone()) - .await?; - - if uuids.is_empty() { - return Ok(()); - } - - let tasks = uuids - .iter() - .map(|&uuid| self.update_handle.dump(uuid, temp_dump_path.clone())) - .collect::>(); - - futures::future::try_join_all(tasks).await?; - - let dump_dir = self.dump_path.clone(); - let dump_path = self.dump_path.join(format!("{}.dump", self.db_name)); - let dump_path = spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; - let temp_dump_file_path = temp_dump_file.path().to_owned(); - compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; - temp_dump_file.persist(&dump_path)?; - Ok(dump_path) - }) - .await??; - - info!("Created dump in {:?}.", dump_path); - */ + tokio::fs::write(path, serde_json::to_string(self)?).await?; Ok(()) } } -*/ + +/// Generate uid from creation date +fn generate_uid() -> String { + Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() +} + +pub async fn perform_dump(index_controller: &IndexController, dump_path: PathBuf) -> anyhow::Result { + info!("Performing dump."); + + let dump_dir = dump_path.clone(); + let uid = generate_uid(); + fs::create_dir_all(&dump_dir).await?; + let temp_dump_dir = spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let uuids = index_controller.uuid_resolver.list().await?; + // maybe we could just keep the vec as-is + let uuids: HashSet<(String, Uuid)> = uuids.into_iter().collect(); + + if uuids.is_empty() { + return Ok(uid); + } + + let indexes = index_controller.list_indexes().await?; + + // we create one directory by index + for meta in indexes.iter() { + tokio::fs::create_dir(temp_dump_path.join(&meta.uid)).await?; + } + + let metadata = Metadata::new(indexes, env!("CARGO_PKG_VERSION").to_string()); + metadata.to_path(&temp_dump_path).await?; + + index_controller.update_handle.dump(uuids, temp_dump_path.clone()).await?; + let dump_dir = dump_path.clone(); + let dump_path = dump_path.join(format!("{}.dump", uid)); + let dump_path = spawn_blocking(move || -> anyhow::Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; + let temp_dump_file_path = temp_dump_file.path().to_owned(); + compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; + temp_dump_file.persist(&dump_path)?; + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(uid) +} /* /// Write Settings in `settings.json` file at provided `dir_path` fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> { - let path = dir_path.join("settings.json"); - let file = File::create(path)?; +let path = dir_path.join("settings.json"); +let file = File::create(path)?; - serde_json::to_writer(file, settings)?; +serde_json::to_writer(file, settings)?; - Ok(()) +Ok(()) } */ diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index ecf71b62b..623b42ddc 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -122,8 +122,8 @@ impl IndexActor { Snapshot { uuid, path, ret } => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } - Dump { uuid, path, ret } => { - let _ = ret.send(self.handle_dump(uuid, path).await); + Dump { uid, uuid, path, ret } => { + let _ = ret.send(self.handle_dump(&uid, uuid, path).await); } GetStats { uuid, ret } => { let _ = ret.send(self.handle_get_stats(uuid).await); @@ -312,24 +312,52 @@ impl IndexActor { Ok(()) } - async fn handle_dump(&self, uuid: Uuid, mut path: PathBuf) -> IndexResult<()> { + /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the + /// documents and all the settings. + async fn handle_dump(&self, uid: &str, uuid: Uuid, path: PathBuf) -> IndexResult<()> { use tokio::fs::create_dir_all; + use std::io::prelude::*; - path.push("indexes"); create_dir_all(&path) .await .map_err(|e| IndexError::Error(e.into()))?; if let Some(index) = self.store.get(uuid).await? { - let mut index_path = path.join(format!("index-{}", uuid)); - create_dir_all(&index_path) - .await - .map_err(|e| IndexError::Error(e.into()))?; - index_path.push("data.mdb"); + let documents_path = path.join(uid).join("documents.jsonl"); + let settings_path = path.join(uid).join("settings.json"); + spawn_blocking(move || -> anyhow::Result<()> { + // first we dump all the documents + let file = File::create(documents_path)?; + let mut file = std::io::BufWriter::new(file); + // Get write txn to wait for ongoing write transaction before dump. - let _txn = index.write_txn()?; - index.env.copy_to_path(index_path, CompactionOption::Enabled)?; + let txn = index.write_txn()?; + let documents_ids = index.documents_ids(&txn)?; + // TODO: TAMO: calling this function here can consume **a lot** of RAM, we should + // use some kind of iterators -> waiting for a milli release + let documents = index.documents(&txn, documents_ids)?; + + let fields_ids_map = index.fields_ids_map(&txn)?; + // we want to save **all** the fields in the dump. + let fields_to_dump: Vec = fields_ids_map.iter().map(|(id, _)| id).collect(); + + for (_doc_id, document) in documents { + let json = milli::obkv_to_json(&fields_to_dump, &fields_ids_map, document)?; + file.write_all(serde_json::to_string(&json)?.as_bytes())?; + file.write_all(b"\n")?; + } + + + // then we dump all the settings + let file = File::create(settings_path)?; + let mut file = std::io::BufWriter::new(file); + let settings = index.settings()?; + + file.write_all(serde_json::to_string(&settings)?.as_bytes())?; + file.write_all(b"\n")?; + + Ok(()) }) .await diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 26aa189d0..64b63e5f0 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -136,9 +136,9 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()> { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Dump { uuid, path, ret }; + let msg = IndexMsg::Dump { uid, uuid, path, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 714a30ecc..37faa1e31 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -61,6 +61,7 @@ pub enum IndexMsg { ret: oneshot::Sender>, }, Dump { + uid: String, uuid: Uuid, path: PathBuf, ret: oneshot::Sender>, diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index e06658ff8..0145a33d9 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -97,7 +97,7 @@ pub trait IndexActorHandle { index_settings: IndexSettings, ) -> IndexResult; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; - async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; + async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()>; async fn get_index_stats(&self, uuid: Uuid) -> IndexResult; } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index ebcc9ed76..6ea42c73d 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, path::PathBuf}; use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -378,6 +378,13 @@ impl IndexController { Ok(stats) } + pub async fn dump(&self, path: PathBuf) -> anyhow::Result { + eprintln!("index_controller::mod called"); + let res = dump::perform_dump(self, path).await?; + eprintln!("index_controller::mod finished"); + Ok(res) + } + pub async fn get_all_stats(&self) -> anyhow::Result { let update_infos = self.update_handle.get_info().await?; let mut database_size = self.get_uuids_size().await? + update_infos.size; diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index fee907001..64794bc6f 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -235,11 +235,11 @@ where Ok(()) } - async fn handle_dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + async fn handle_dump(&self, uuids: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()> { let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - update_store.dump(&uuids, &path)?; + update_store.dump(&uuids, path.to_path_buf())?; // Perform the snapshot of each index concurently. Only a third of the capabilities of // the index actor at a time not to put too much pressure on the index actor @@ -247,7 +247,7 @@ where let handle = &index_handle; let mut stream = futures::stream::iter(uuids.iter()) - .map(|&uuid| handle.dump(uuid, path.clone())) + .map(|(uid, uuid)| handle.dump(uid.clone(), *uuid, path.clone())) .buffer_unordered(CONCURRENT_INDEX_MSG / 3); Handle::current().block_on(async { diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 09a242377..a497a3c5c 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -71,16 +71,16 @@ where receiver.await.expect("update actor killed.") } - async fn get_info(&self) -> Result { + async fn dump(&self, uuids: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::GetInfo { ret }; + let msg = UpdateMsg::Dump { uuids, path, ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } - async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + async fn get_info(&self) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Dump { uuids, path, ret }; + let msg = UpdateMsg::GetInfo { ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 37df2af32..4103ca121 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -32,7 +32,7 @@ pub enum UpdateMsg { ret: oneshot::Sender>, }, Dump { - uuids: HashSet, + uuids: HashSet<(String, Uuid)>, path: PathBuf, ret: oneshot::Sender>, }, diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index eeca6629f..05b793e45 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -40,7 +40,7 @@ pub trait UpdateActorHandle { async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; async fn snapshot(&self, uuid: HashSet, path: PathBuf) -> Result<()>; - async fn dump(&self, uuid: HashSet, path: PathBuf) -> Result<()>; + async fn dump(&self, uuid: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()>; async fn get_info(&self) -> Result; async fn update( &self, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 27018764f..d767dfa93 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -1,4 +1,4 @@ -use std::borrow::Cow; +use std::{borrow::Cow, path::PathBuf}; use std::collections::{BTreeMap, HashSet}; use std::convert::TryInto; use std::fs::{copy, create_dir_all, remove_file, File}; @@ -294,6 +294,7 @@ impl UpdateStore { result: UpdateStatus, index_uuid: Uuid, ) -> heed::Result<()> { + // TODO: TAMO: load already processed updates let mut wtxn = self.env.write_txn()?; let (_global_id, update_id) = self.next_update_id(&mut wtxn, index_uuid)?; self.updates.remap_key_type::().put(&mut wtxn, &(index_uuid, update_id), &result)?; @@ -516,31 +517,34 @@ impl UpdateStore { Ok(()) } - pub fn dump(&self, uuids: &HashSet, path: impl AsRef) -> anyhow::Result<()> { + pub fn dump(&self, uuids: &HashSet<(String, Uuid)>, path: PathBuf) -> anyhow::Result<()> { + use std::io::prelude::*; let state_lock = self.state.write(); - state_lock.swap(State::Snapshoting); // TODO: rename the state + state_lock.swap(State::Snapshoting); // TODO: TAMO rename the state somehow let txn = self.env.write_txn()?; - let update_path = path.as_ref().join("updates"); - create_dir_all(&update_path)?; + for (uid, uuid) in uuids.iter() { + let file = File::create(path.join(uid).join("updates.jsonl"))?; + let mut file = std::io::BufWriter::new(file); - // acquire write lock to prevent further writes during dump - create_dir_all(&update_path)?; - let db_path = update_path.join("data.mdb"); + for update in &self.list(*uuid)? { + serde_json::to_writer(&mut file, update)?; + file.write_all(b"\n")?; + } + } - // TODO: everything - // create db dump - self.env.copy_to_path(&db_path, CompactionOption::Enabled)?; + // TODO: TAMO: the updates + // already processed updates seems to works, but I've not tried with currently running updates - let update_files_path = update_path.join("update_files"); + let update_files_path = path.join("update_files"); create_dir_all(&update_files_path)?; let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); for entry in pendings { let ((_, uuid, _), pending) = entry?; - if uuids.contains(&uuid) { + if uuids.iter().any(|(_, id)| id == &uuid) { if let Some(path) = pending.decode()?.content_path() { let name = path.file_name().unwrap(); let to = update_files_path.join(name); diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index df83ceba9..253326276 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -41,9 +41,6 @@ impl UuidResolverActor { Some(SnapshotRequest { path, ret }) => { let _ = ret.send(self.handle_snapshot(path).await); } - Some(DumpRequest { path, ret }) => { - let _ = ret.send(self.handle_dump(path).await); - } Some(GetSize { ret }) => { let _ = ret.send(self.handle_get_size().await); } @@ -85,10 +82,6 @@ impl UuidResolverActor { self.store.snapshot(path).await } - async fn handle_dump(&self, path: PathBuf) -> Result> { - self.store.dump(path).await - } - async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> { if !is_index_uid_valid(&uid) { return Err(UuidError::BadlyFormatted(uid)); diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index d9e9a20fc..db4c482bd 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -68,7 +68,6 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - /// TODO: we should merge this function with the dump function async fn snapshot(&self, path: PathBuf) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::SnapshotRequest { path, ret }; @@ -78,15 +77,6 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - async fn dump(&self, path: PathBuf) -> Result> { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::DumpRequest { path, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - async fn get_size(&self) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::GetSize { ret }; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index 78f62eea2..a72bf0587 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -31,10 +31,6 @@ pub enum UuidResolveMsg { path: PathBuf, ret: oneshot::Sender>>, }, - DumpRequest { - path: PathBuf, - ret: oneshot::Sender>>, - }, GetSize { ret: oneshot::Sender>, }, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index aca730db9..0cbb2895b 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -32,7 +32,6 @@ pub trait UuidResolverHandle { async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; async fn snapshot(&self, path: PathBuf) -> Result>; - async fn dump(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 917e0b4a5..a781edcba 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -21,7 +21,6 @@ pub trait UuidStore { async fn list(&self) -> Result>; async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; async fn snapshot(&self, path: PathBuf) -> Result>; - async fn dump(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } @@ -114,8 +113,6 @@ impl HeedUuidStore { Ok(()) } - // TODO: we should merge this function and the following function for the dump. it's exactly - // the same code pub fn snapshot(&self, mut path: PathBuf) -> Result> { let env = self.env.clone(); let db = self.db; @@ -138,28 +135,6 @@ impl HeedUuidStore { Ok(entries) } - pub fn dump(&self, mut path: PathBuf) -> Result> { - let env = self.env.clone(); - let db = self.db; - // Write transaction to acquire a lock on the database. - let txn = env.write_txn()?; - let mut entries = HashSet::new(); - for entry in db.iter(&txn)? { - let (_, uuid) = entry?; - let uuid = Uuid::from_slice(uuid)?; - entries.insert(uuid); - } - - // only perform dump if there are indexes - if !entries.is_empty() { - path.push("index_uuids"); - create_dir_all(&path).unwrap(); - path.push("data.mdb"); - env.copy_to_path(path, CompactionOption::Enabled)?; - } - Ok(entries) - } - pub fn get_size(&self) -> Result { Ok(self.env.size()) } @@ -197,11 +172,6 @@ impl UuidStore for HeedUuidStore { tokio::task::spawn_blocking(move || this.snapshot(path)).await? } - async fn dump(&self, path: PathBuf) -> Result> { - let this = self.clone(); - tokio::task::spawn_blocking(move || this.dump(path)).await? - } - async fn get_size(&self) -> Result { self.get_size() } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index fd5cf6786..e19037482 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -46,8 +46,8 @@ macro_rules! create_app { .configure(synonym::services) .configure(health::services) .configure(stats::services) - .configure(key::services); - //.configure(routes::dump::services); + .configure(key::services) + .configure(dump::services); #[cfg(feature = "mini-dashboard")] let app = if $enable_frontend { let generated = dashboard::generate(); diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index c46b0e502..410b817b8 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -1,14 +1,10 @@ -use std::fs::File; -use std::path::Path; +use actix_web::{post, get, web}; +use actix_web::HttpResponse; +use serde::{Serialize, Deserialize}; -use actix_web::{get, post}; -use actix_web::{HttpResponse, web}; -use serde::{Deserialize, Serialize}; - -use crate::dump::{DumpInfo, DumpStatus, compressed_dumps_dir, init_dump_process}; -use crate::Data; -use crate::error::{Error, ResponseError}; +use crate::error::ResponseError; use crate::helpers::Authentication; +use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(trigger_dump) @@ -19,7 +15,10 @@ pub fn services(cfg: &mut web::ServiceConfig) { async fn trigger_dump( data: web::Data, ) -> Result { - todo!() + eprintln!("dump started"); + let res = data.dump().await?; + + Ok(HttpResponse::Ok().body(res)) } #[derive(Debug, Serialize)] @@ -30,13 +29,13 @@ struct DumpStatusResponse { #[derive(Deserialize)] struct DumpParam { - dump_uid: String, + _dump_uid: String, } #[get("/dumps/{dump_uid}/status", wrap = "Authentication::Private")] async fn get_dump_status( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index aaf13613a..999c4f881 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -9,7 +9,7 @@ pub mod search; pub mod settings; pub mod stats; pub mod synonym; -//pub mod dump; +pub mod dump; #[derive(Deserialize)] pub struct IndexParam { From efca63f9cea84d8b98255ea7b2a333dabf63f924 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 10 May 2021 20:25:09 +0200 Subject: [PATCH 317/522] [WIP] rebase on main --- meilisearch-http/src/data/mod.rs | 11 +- .../src/index_controller/dump_actor/actor.rs | 200 ++++++++++++++++++ .../dump_actor/handle_impl.rs | 41 ++++ .../index_controller/dump_actor/message.rs | 15 ++ .../{dump => dump_actor}/mod.rs | 151 +++++++------ .../{dump => dump_actor}/v1.rs | 0 .../{dump => dump_actor}/v2.rs | 0 .../src/index_controller/index_actor/actor.rs | 4 +- meilisearch-http/src/index_controller/mod.rs | 27 ++- meilisearch-http/src/routes/dump.rs | 19 +- 10 files changed, 381 insertions(+), 87 deletions(-) create mode 100644 meilisearch-http/src/index_controller/dump_actor/actor.rs create mode 100644 meilisearch-http/src/index_controller/dump_actor/handle_impl.rs create mode 100644 meilisearch-http/src/index_controller/dump_actor/message.rs rename meilisearch-http/src/index_controller/{dump => dump_actor}/mod.rs (63%) rename meilisearch-http/src/index_controller/{dump => dump_actor}/v1.rs (100%) rename meilisearch-http/src/index_controller/{dump => dump_actor}/v2.rs (100%) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 39cfed626..008065d74 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -4,8 +4,7 @@ use std::sync::Arc; use sha2::Digest; use crate::index::{Checked, Settings}; -use crate::index_controller::{IndexController, IndexStats, Stats}; -use crate::index_controller::{IndexMetadata, IndexSettings}; +use crate::index_controller::{IndexController, IndexStats, Stats, DumpInfo, IndexMetadata, IndexSettings}; use crate::option::Opt; pub mod search; @@ -108,8 +107,12 @@ impl Data { Ok(self.index_controller.get_all_stats().await?) } - pub async fn dump(&self) -> anyhow::Result { - Ok(self.index_controller.dump(self.options.dumps_dir.clone()).await?) + pub async fn create_dump(&self) -> anyhow::Result { + Ok(self.index_controller.create_dump().await?) + } + + pub async fn dump_status(&self, uid: String) -> anyhow::Result { + Ok(self.index_controller.dump_info(uid).await?) } #[inline] diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs new file mode 100644 index 000000000..b41ddadcf --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -0,0 +1,200 @@ +use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus}; +use crate::helpers::compression; +use crate::index_controller::{index_actor, update_actor, uuid_resolver, IndexMetadata}; +use chrono::Utc; +use log::{error, info, warn}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::sync::{mpsc, Mutex}; +use uuid::Uuid; + +pub struct DumpActor { + inbox: mpsc::Receiver, + inner: InnerDump, +} + +#[derive(Clone)] +struct InnerDump { + pub uuid_resolver: UuidResolver, + pub index: Index, + pub update: Update, + pub dump_path: PathBuf, + pub dump_info: Arc>>, +} + +/// Generate uid from creation date +fn generate_uid() -> String { + Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() +} + +impl DumpActor +where + UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, + Index: index_actor::IndexActorHandle + Send + Sync + Clone + 'static, + Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, +{ + pub fn new( + inbox: mpsc::Receiver, + uuid_resolver: UuidResolver, + index: Index, + update: Update, + dump_path: impl AsRef, + ) -> Self { + Self { + inbox, + inner: InnerDump { + uuid_resolver, + index, + update, + dump_path: dump_path.as_ref().into(), + dump_info: Arc::new(Mutex::new(None)), + }, + } + } + + pub async fn run(mut self) { + use DumpMsg::*; + + info!("Started dump actor."); + + loop { + match self.inbox.recv().await { + Some(CreateDump { ret }) => { + let _ = ret.send(self.inner.clone().handle_create_dump().await); + } + Some(DumpInfo { ret, uid }) => { + let _ = ret.send(self.inner.handle_dump_info(uid).await); + } + None => break, + } + } + + error!("Dump actor stopped."); + } +} + +impl InnerDump +where + UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, + Index: index_actor::IndexActorHandle + Send + Sync + Clone + 'static, + Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, +{ + async fn handle_create_dump(self) -> DumpResult { + if self.is_running().await { + return Err(DumpError::DumpAlreadyRunning); + } + let uid = generate_uid(); + let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress); + *self.dump_info.lock().await = Some(info.clone()); + + let this = self.clone(); + + tokio::task::spawn(async move { + match this.perform_dump(uid).await { + Ok(()) => { + if let Some(ref mut info) = *self.dump_info.lock().await { + info.done(); + } else { + warn!("dump actor was in an inconsistant state"); + } + info!("Dump succeed"); + } + Err(e) => { + if let Some(ref mut info) = *self.dump_info.lock().await { + info.with_error(e.to_string()); + } else { + warn!("dump actor was in an inconsistant state"); + } + error!("Dump failed: {}", e); + } + }; + }); + + Ok(info) + } + + async fn perform_dump(self, uid: String) -> anyhow::Result<()> { + info!("Performing dump."); + + let dump_dir = self.dump_path.clone(); + tokio::fs::create_dir_all(&dump_dir).await?; + let temp_dump_dir = + tokio::task::spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let uuids = self.uuid_resolver.list().await?; + // maybe we could just keep the vec as-is + let uuids: HashSet<(String, Uuid)> = uuids.into_iter().collect(); + + if uuids.is_empty() { + return Ok(()); + } + + let indexes = self.list_indexes().await?; + + // we create one directory by index + for meta in indexes.iter() { + tokio::fs::create_dir(temp_dump_path.join(&meta.uid)).await?; + } + + let metadata = super::Metadata::new(indexes, env!("CARGO_PKG_VERSION").to_string()); + metadata.to_path(&temp_dump_path).await?; + + self.update.dump(uuids, temp_dump_path.clone()).await?; + + let dump_dir = self.dump_path.clone(); + let dump_path = self.dump_path.join(format!("{}.dump", uid)); + let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; + let temp_dump_file_path = temp_dump_file.path().to_owned(); + compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; + temp_dump_file.persist(&dump_path)?; + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(()) + } + + async fn list_indexes(&self) -> anyhow::Result> { + let uuids = self.uuid_resolver.list().await?; + + let mut ret = Vec::new(); + + for (uid, uuid) in uuids { + let meta = self.index.get_index_meta(uuid).await?; + let meta = IndexMetadata { + uuid, + name: uid.clone(), + uid, + meta, + }; + ret.push(meta); + } + + Ok(ret) + } + + async fn handle_dump_info(&self, uid: String) -> DumpResult { + match &*self.dump_info.lock().await { + None => Err(DumpError::DumpDoesNotExist(uid)), + Some(DumpInfo { uid: ref s, .. }) if &uid != s => Err(DumpError::DumpDoesNotExist(uid)), + Some(info) => Ok(info.clone()), + } + } + + async fn is_running(&self) -> bool { + matches!( + *self.dump_info.lock().await, + Some(DumpInfo { + status: DumpStatus::InProgress, + .. + }) + ) + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs new file mode 100644 index 000000000..601c97c01 --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs @@ -0,0 +1,41 @@ +use std::path::{Path}; +use actix_web::web::Bytes; +use tokio::sync::{mpsc, oneshot}; +use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg, DumpResult}; + +#[derive(Clone)] +pub struct DumpActorHandleImpl { + sender: mpsc::Sender, +} + +#[async_trait::async_trait] +impl DumpActorHandle for DumpActorHandleImpl { + async fn create_dump(&self) -> DumpResult { + let (ret, receiver) = oneshot::channel(); + let msg = DumpMsg::CreateDump { ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } + + async fn dump_info(&self, uid: String) -> DumpResult { + let (ret, receiver) = oneshot::channel(); + let msg = DumpMsg::DumpInfo { ret, uid }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } +} + +impl DumpActorHandleImpl { + pub fn new( + path: impl AsRef, + uuid_resolver: crate::index_controller::uuid_resolver::UuidResolverHandleImpl, + index: crate::index_controller::index_actor::IndexActorHandleImpl, + update: crate::index_controller::update_actor::UpdateActorHandleImpl, + ) -> anyhow::Result { + let (sender, receiver) = mpsc::channel(10); + let actor = DumpActor::new(receiver, uuid_resolver, index, update, path); + + tokio::task::spawn(actor.run()); + Ok(Self { sender }) + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/message.rs b/meilisearch-http/src/index_controller/dump_actor/message.rs new file mode 100644 index 000000000..14409afbb --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/message.rs @@ -0,0 +1,15 @@ +use tokio::sync::oneshot; + +use super::{DumpResult, DumpInfo}; + + +pub enum DumpMsg { + CreateDump { + ret: oneshot::Sender>, + }, + DumpInfo { + uid: String, + ret: oneshot::Sender>, + }, +} + diff --git a/meilisearch-http/src/index_controller/dump/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs similarity index 63% rename from meilisearch-http/src/index_controller/dump/mod.rs rename to meilisearch-http/src/index_controller/dump_actor/mod.rs index a44d4235b..f57c27c59 100644 --- a/meilisearch-http/src/index_controller/dump/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,23 +1,48 @@ mod v1; mod v2; +mod handle_impl; +mod actor; +mod message; -use std::{collections::HashSet, fs::{File}, path::{Path, PathBuf}, sync::Arc}; +use std::{ + fs::File, + path::Path, + sync::Arc, +}; +#[cfg(test)] +use mockall::automock; use anyhow::bail; -use chrono::Utc; +use thiserror::Error; use heed::EnvOpenOptions; use log::{error, info}; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; +use serde_json::json; use tempfile::TempDir; -use tokio::task::spawn_blocking; -use tokio::fs; -use uuid::Uuid; -use super::{IndexController, IndexMetadata, update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle}; +use super::IndexMetadata; +use crate::helpers::compression; use crate::index::Index; use crate::index_controller::uuid_resolver; -use crate::helpers::compression; + +pub use handle_impl::*; +pub use actor::DumpActor; +pub use message::DumpMsg; + +pub type DumpResult = std::result::Result; + +#[derive(Error, Debug)] +pub enum DumpError { + #[error("error with index: {0}")] + Error(#[from] anyhow::Error), + #[error("Heed error: {0}")] + HeedError(#[from] heed::Error), + #[error("dump already running")] + DumpAlreadyRunning, + #[error("dump `{0}` does not exist")] + DumpDoesNotExist(String), +} #[derive(Debug, Serialize, Deserialize, Copy, Clone)] enum DumpVersion { @@ -29,7 +54,12 @@ impl DumpVersion { const CURRENT: Self = Self::V2; /// Select the good importation function from the `DumpVersion` of metadata - pub fn import_index(self, size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { + pub fn import_index( + self, + size: usize, + dump_path: &Path, + index_path: &Path, + ) -> anyhow::Result<()> { match self { Self::V1 => v1::import_index(size, dump_path, index_path), Self::V2 => v2::import_index(size, dump_path, index_path), @@ -37,6 +67,19 @@ impl DumpVersion { } } +#[async_trait::async_trait] +#[cfg_attr(test, automock)] +pub trait DumpActorHandle { + /// Start the creation of a dump + /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump] + async fn create_dump(&self) -> DumpResult; + + /// Return the status of an already created dump + /// Implementation: [handle_impl::DumpActorHandleImpl::dump_status] + async fn dump_info(&self, uid: String) -> DumpResult; +} + + #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Metadata { @@ -74,66 +117,46 @@ impl Metadata { } } -/// Generate uid from creation date -fn generate_uid() -> String { - Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() +#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +#[serde(rename_all = "snake_case")] +pub enum DumpStatus { + Done, + InProgress, + Failed, } -pub async fn perform_dump(index_controller: &IndexController, dump_path: PathBuf) -> anyhow::Result { - info!("Performing dump."); +#[derive(Debug, Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct DumpInfo { + pub uid: String, + pub status: DumpStatus, + #[serde(skip_serializing_if = "Option::is_none", flatten)] + pub error: Option, +} - let dump_dir = dump_path.clone(); - let uid = generate_uid(); - fs::create_dir_all(&dump_dir).await?; - let temp_dump_dir = spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); - - let uuids = index_controller.uuid_resolver.list().await?; - // maybe we could just keep the vec as-is - let uuids: HashSet<(String, Uuid)> = uuids.into_iter().collect(); - - if uuids.is_empty() { - return Ok(uid); +impl DumpInfo { + pub fn new(uid: String, status: DumpStatus) -> Self { + Self { + uid, + status, + error: None, + } } - let indexes = index_controller.list_indexes().await?; - - // we create one directory by index - for meta in indexes.iter() { - tokio::fs::create_dir(temp_dump_path.join(&meta.uid)).await?; + pub fn with_error(&mut self, error: String) { + self.status = DumpStatus::Failed; + self.error = Some(json!(error)); } - let metadata = Metadata::new(indexes, env!("CARGO_PKG_VERSION").to_string()); - metadata.to_path(&temp_dump_path).await?; + pub fn done(&mut self) { + self.status = DumpStatus::Done; + } - index_controller.update_handle.dump(uuids, temp_dump_path.clone()).await?; - let dump_dir = dump_path.clone(); - let dump_path = dump_path.join(format!("{}.dump", uid)); - let dump_path = spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; - let temp_dump_file_path = temp_dump_file.path().to_owned(); - compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; - temp_dump_file.persist(&dump_path)?; - Ok(dump_path) - }) - .await??; - - info!("Created dump in {:?}.", dump_path); - - Ok(uid) + pub fn dump_already_in_progress(&self) -> bool { + self.status == DumpStatus::InProgress + } } -/* -/// Write Settings in `settings.json` file at provided `dir_path` -fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> { -let path = dir_path.join("settings.json"); -let file = File::create(path)?; - -serde_json::to_writer(file, settings)?; - -Ok(()) -} -*/ pub fn load_dump( db_path: impl AsRef, @@ -185,12 +208,18 @@ pub fn load_dump( let index_path = db_path.join(&format!("indexes/index-{}", uuid)); // let update_path = db_path.join(&format!("updates/updates-{}", uuid)); // TODO: add the update db - info!("Importing dump from {} into {}...", dump_path.display(), index_path.display()); - metadata.dump_version.import_index(size, &dump_path, &index_path).unwrap(); + info!( + "Importing dump from {} into {}...", + dump_path.display(), + index_path.display() + ); + metadata + .dump_version + .import_index(size, &dump_path, &index_path) + .unwrap(); info!("Dump importation from {} succeed", dump_path.display()); } - info!("Dump importation from {} succeed", dump_path.display()); Ok(()) } diff --git a/meilisearch-http/src/index_controller/dump/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs similarity index 100% rename from meilisearch-http/src/index_controller/dump/v1.rs rename to meilisearch-http/src/index_controller/dump_actor/v1.rs diff --git a/meilisearch-http/src/index_controller/dump/v2.rs b/meilisearch-http/src/index_controller/dump_actor/v2.rs similarity index 100% rename from meilisearch-http/src/index_controller/dump/v2.rs rename to meilisearch-http/src/index_controller/dump_actor/v2.rs diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 623b42ddc..ca23663b7 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -315,8 +315,8 @@ impl IndexActor { /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the /// documents and all the settings. async fn handle_dump(&self, uid: &str, uuid: Uuid, path: PathBuf) -> IndexResult<()> { - use tokio::fs::create_dir_all; use std::io::prelude::*; + use tokio::fs::create_dir_all; create_dir_all(&path) .await @@ -348,7 +348,6 @@ impl IndexActor { file.write_all(b"\n")?; } - // then we dump all the settings let file = File::create(settings_path)?; let mut file = std::io::BufWriter::new(file); @@ -357,7 +356,6 @@ impl IndexActor { file.write_all(serde_json::to_string(&settings)?.as_bytes())?; file.write_all(b"\n")?; - Ok(()) }) .await diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 6ea42c73d..d1bb5e170 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, path::PathBuf}; +use std::collections::BTreeMap; use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -15,6 +15,8 @@ use tokio::time::sleep; use uuid::Uuid; pub use updates::*; +pub use dump_actor::{DumpInfo, DumpStatus}; +use dump_actor::DumpActorHandle; use index_actor::IndexActorHandle; use snapshot::{SnapshotService, load_snapshot}; use update_actor::UpdateActorHandle; @@ -23,11 +25,11 @@ use uuid_resolver::{UuidError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; -use self::dump::load_dump; +use dump_actor::load_dump; mod index_actor; mod snapshot; -mod dump; +mod dump_actor; mod update_actor; mod update_handler; mod updates; @@ -63,10 +65,12 @@ pub struct IndexStats { pub fields_distribution: FieldsDistribution, } +#[derive(Clone)] pub struct IndexController { uuid_resolver: uuid_resolver::UuidResolverHandleImpl, index_handle: index_actor::IndexActorHandleImpl, update_handle: update_actor::UpdateActorHandleImpl, + dump_handle: dump_actor::DumpActorHandleImpl, } #[derive(Serialize)] @@ -108,6 +112,7 @@ impl IndexController { &path, update_store_size, )?; + let dump_handle = dump_actor::DumpActorHandleImpl::new(&options.dumps_dir, uuid_resolver.clone(), index_handle.clone(), update_handle.clone())?; if options.schedule_snapshot { let snapshot_service = SnapshotService::new( @@ -129,6 +134,7 @@ impl IndexController { uuid_resolver, index_handle, update_handle, + dump_handle, }) } @@ -378,13 +384,6 @@ impl IndexController { Ok(stats) } - pub async fn dump(&self, path: PathBuf) -> anyhow::Result { - eprintln!("index_controller::mod called"); - let res = dump::perform_dump(self, path).await?; - eprintln!("index_controller::mod finished"); - Ok(res) - } - pub async fn get_all_stats(&self) -> anyhow::Result { let update_infos = self.update_handle.get_info().await?; let mut database_size = self.get_uuids_size().await? + update_infos.size; @@ -410,6 +409,14 @@ impl IndexController { indexes, }) } + + pub async fn create_dump(&self) -> anyhow::Result { + Ok(self.dump_handle.create_dump().await?) + } + + pub async fn dump_info(&self, uid: String) -> anyhow::Result { + Ok(self.dump_handle.dump_info(uid).await?) + } } pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 410b817b8..e6be4ca93 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -7,18 +7,17 @@ use crate::helpers::Authentication; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(trigger_dump) + cfg.service(create_dump) .service(get_dump_status); } #[post("/dumps", wrap = "Authentication::Private")] -async fn trigger_dump( +async fn create_dump( data: web::Data, ) -> Result { - eprintln!("dump started"); - let res = data.dump().await?; + let res = data.create_dump().await?; - Ok(HttpResponse::Ok().body(res)) + Ok(HttpResponse::Ok().json(res)) } #[derive(Debug, Serialize)] @@ -29,13 +28,15 @@ struct DumpStatusResponse { #[derive(Deserialize)] struct DumpParam { - _dump_uid: String, + dump_uid: String, } #[get("/dumps/{dump_uid}/status", wrap = "Authentication::Private")] async fn get_dump_status( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() + let res = data.dump_status(path.dump_uid.clone()).await?; + + Ok(HttpResponse::Ok().json(res)) } From 24192fc5504894c0aa61576cbc6e3ea2a50c041b Mon Sep 17 00:00:00 2001 From: tamo Date: Wed, 5 May 2021 18:03:21 +0200 Subject: [PATCH 318/522] fix tests --- meilisearch-http/src/index_controller/dump_actor/v1.rs | 1 - meilisearch-http/src/index_controller/dump_actor/v2.rs | 1 - meilisearch-http/src/index_controller/index_actor/mod.rs | 4 ++-- meilisearch-http/src/index_controller/update_actor/actor.rs | 2 +- .../src/index_controller/update_actor/update_store.rs | 6 ++++-- meilisearch-http/tests/settings/get_settings.rs | 2 -- 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs index 3a20299f3..f22120849 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v1.rs @@ -89,7 +89,6 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: // extract `settings.json` file and import content let settings = import_settings(&dump_path)?; - dbg!(&settings); let settings: index_controller::Settings = settings.into(); let update_builder = UpdateBuilder::new(0); index.update_settings(&settings, update_builder)?; diff --git a/meilisearch-http/src/index_controller/dump_actor/v2.rs b/meilisearch-http/src/index_controller/dump_actor/v2.rs index 7b9a56772..5c5e5fb2d 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v2.rs @@ -25,7 +25,6 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: let settings = import_settings(&dump_path)?; let update_builder = UpdateBuilder::new(0); index.update_settings(&settings, update_builder)?; - dbg!(settings); let update_builder = UpdateBuilder::new(1); let file = File::open(&dump_path.join("documents.jsonl"))?; diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 0145a33d9..cf6a81223 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -178,8 +178,8 @@ mod test { self.as_ref().snapshot(uuid, path).await } - async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { - self.as_ref().dump(uuid, path).await + async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + self.as_ref().dump(uid, uuid, path).await } async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 64794bc6f..fe4458acd 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -241,7 +241,7 @@ where tokio::task::spawn_blocking(move || -> anyhow::Result<()> { update_store.dump(&uuids, path.to_path_buf())?; - // Perform the snapshot of each index concurently. Only a third of the capabilities of + // Perform the dump of each index concurently. Only a third of the capabilities of // the index actor at a time not to put too much pressure on the index actor let path = &path; let handle = &index_handle; diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index d767dfa93..f3d7dfd0a 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -11,6 +11,7 @@ use arc_swap::ArcSwap; use heed::types::{ByteSlice, OwnedType, SerdeJson}; use heed::zerocopy::U64; use heed::{BytesDecode, BytesEncode, CompactionOption, Database, Env, EnvOpenOptions}; +use log::error; use parking_lot::{Mutex, MutexGuard}; use tokio::runtime::Handle; use tokio::sync::mpsc; @@ -77,6 +78,7 @@ pub enum State { Idle, Processing(Uuid, Processing), Snapshoting, + Dumping, } impl<'a> BytesEncode<'a> for NextIdCodec { @@ -227,7 +229,7 @@ impl UpdateStore { match res { Ok(Some(_)) => (), Ok(None) => break, - Err(e) => eprintln!("error while processing update: {}", e), + Err(e) => error!("error while processing update: {}", e), } } // the ownership on the arc has been taken, we need to exit. @@ -520,7 +522,7 @@ impl UpdateStore { pub fn dump(&self, uuids: &HashSet<(String, Uuid)>, path: PathBuf) -> anyhow::Result<()> { use std::io::prelude::*; let state_lock = self.state.write(); - state_lock.swap(State::Snapshoting); // TODO: TAMO rename the state somehow + state_lock.swap(State::Dumping); let txn = self.env.write_txn()?; diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index a39dd54e9..4230e19f8 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -82,9 +82,7 @@ async fn reset_all_settings() { assert_eq!(response["searchableAttributes"], json!(["bar"])); assert_eq!(response["stopWords"], json!(["the"])); - eprintln!("BEFORE"); index.delete_settings().await; - eprintln!("AFTER"); index.wait_update_id(1).await; let (response, code) = index.settings().await; From 956012da95c78d83a8832e8e2e4dc27d506ee58c Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 5 May 2021 19:06:07 +0200 Subject: [PATCH 319/522] fix dump lock --- .../index_controller/update_actor/actor.rs | 24 +++-------------- .../update_actor/update_store.rs | 26 ++++++++++++++++--- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index fe4458acd..54d068f14 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -239,28 +239,12 @@ where let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - update_store.dump(&uuids, path.to_path_buf())?; - - // Perform the dump of each index concurently. Only a third of the capabilities of - // the index actor at a time not to put too much pressure on the index actor - let path = &path; - let handle = &index_handle; - - let mut stream = futures::stream::iter(uuids.iter()) - .map(|(uid, uuid)| handle.dump(uid.clone(), *uuid, path.clone())) - .buffer_unordered(CONCURRENT_INDEX_MSG / 3); - - Handle::current().block_on(async { - while let Some(res) = stream.next().await { - res?; - } - Ok(()) - }) + update_store.dump(&uuids, path.to_path_buf(), index_handle)?; + Ok(()) }) .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?; - + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?; Ok(()) } diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index f3d7dfd0a..524fefe84 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -16,10 +16,11 @@ use parking_lot::{Mutex, MutexGuard}; use tokio::runtime::Handle; use tokio::sync::mpsc; use uuid::Uuid; +use futures::StreamExt; use super::UpdateMeta; use crate::helpers::EnvSizer; -use crate::index_controller::{IndexActorHandle, updates::*}; +use crate::index_controller::{IndexActorHandle, updates::*, index_actor::CONCURRENT_INDEX_MSG}; #[allow(clippy::upper_case_acronyms)] type BEU64 = U64; @@ -519,7 +520,12 @@ impl UpdateStore { Ok(()) } - pub fn dump(&self, uuids: &HashSet<(String, Uuid)>, path: PathBuf) -> anyhow::Result<()> { + pub fn dump( + &self, + uuids: &HashSet<(String, Uuid)>, + path: PathBuf, + handle: impl IndexActorHandle + ) -> anyhow::Result<()> { use std::io::prelude::*; let state_lock = self.state.write(); state_lock.swap(State::Dumping); @@ -555,7 +561,21 @@ impl UpdateStore { } } - Ok(()) + + // Perform the dump of each index concurently. Only a third of the capabilities of + // the index actor at a time not to put too much pressure on the index actor + let path = &path; + + let mut stream = futures::stream::iter(uuids.iter()) + .map(|(uid, uuid)| handle.dump(uid.clone(), *uuid, path.clone())) + .buffer_unordered(CONCURRENT_INDEX_MSG / 3); + + Handle::current().block_on(async { + while let Some(res) = stream.next().await { + res?; + } + Ok(()) + }) } pub fn get_info(&self) -> anyhow::Result { From 26dcb9e66d5311feea92302e9cb0b57ac0c590f5 Mon Sep 17 00:00:00 2001 From: tamo Date: Thu, 6 May 2021 11:57:42 +0200 Subject: [PATCH 320/522] bump milli version and fix a performance issue for large dumps --- Cargo.lock | 6 ++++-- meilisearch-http/Cargo.toml | 2 +- .../src/index_controller/index_actor/actor.rs | 8 ++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1c109a79..26c53663a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" version = "0.4.0" @@ -1840,8 +1842,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.2.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.2.0#792225eaffce6b3682f9b30b7370b6a547c4757e" +version = "0.2.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.2.1#25f75d4d03732131e6edcf20f4d126210b159d43" dependencies = [ "anyhow", "bstr", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 7ac3ecb38..c9f8d63b7 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.0" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.1" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index ca23663b7..1f0091265 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -333,16 +333,12 @@ impl IndexActor { // Get write txn to wait for ongoing write transaction before dump. let txn = index.write_txn()?; - let documents_ids = index.documents_ids(&txn)?; - // TODO: TAMO: calling this function here can consume **a lot** of RAM, we should - // use some kind of iterators -> waiting for a milli release - let documents = index.documents(&txn, documents_ids)?; - let fields_ids_map = index.fields_ids_map(&txn)?; // we want to save **all** the fields in the dump. let fields_to_dump: Vec = fields_ids_map.iter().map(|(id, _)| id).collect(); - for (_doc_id, document) in documents { + for document in index.all_documents(&txn)? { + let (_doc_id, document) = document?; let json = milli::obkv_to_json(&fields_to_dump, &fields_ids_map, document)?; file.write_all(serde_json::to_string(&json)?.as_bytes())?; file.write_all(b"\n")?; From 5f5402a3abd9fc2bbedf1e3af980eb050a035c3a Mon Sep 17 00:00:00 2001 From: tamo Date: Thu, 6 May 2021 18:12:57 +0200 Subject: [PATCH 321/522] provide a way to access the internal content path of all processing State --- .../src/index_controller/updates.rs | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index a129a25a0..6b5ef345d 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -72,6 +72,10 @@ impl Enqueued { pub fn content_path(&self) -> Option<&Path> { self.content.as_deref() } + + pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { + self.content.as_mut() + } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -87,6 +91,14 @@ impl Processed { pub fn id(&self) -> u64 { self.from.id() } + + pub fn content_path(&self) -> Option<&Path> { + self.from.content_path() + } + + pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { + self.from.content_path_mut() + } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -106,6 +118,14 @@ impl Processing { self.from.meta() } + pub fn content_path(&self) -> Option<&Path> { + self.from.content_path() + } + + pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { + self.from.content_path_mut() + } + pub fn process(self, success: UpdateResult) -> Processed { Processed { success, @@ -135,6 +155,14 @@ impl Aborted { pub fn id(&self) -> u64 { self.from.id() } + + pub fn content_path(&self) -> Option<&Path> { + self.from.content_path() + } + + pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { + self.from.content_path_mut() + } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -150,6 +178,14 @@ impl Failed { pub fn id(&self) -> u64 { self.from.id() } + + pub fn content_path(&self) -> Option<&Path> { + self.from.content_path() + } + + pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { + self.from.content_path_mut() + } } #[derive(Debug, Serialize, Deserialize)] @@ -179,6 +215,26 @@ impl UpdateStatus { _ => None, } } + + pub fn content_path(&self) -> Option<&Path> { + match self { + UpdateStatus::Processing(u) => u.content_path(), + UpdateStatus::Processed(u) => u.content_path(), + UpdateStatus::Aborted(u) => u.content_path(), + UpdateStatus::Failed(u) => u.content_path(), + UpdateStatus::Enqueued(u) => u.content_path(), + } + } + + pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { + match self { + UpdateStatus::Processing(u) => u.content_path_mut(), + UpdateStatus::Processed(u) => u.content_path_mut(), + UpdateStatus::Aborted(u) => u.content_path_mut(), + UpdateStatus::Failed(u) => u.content_path_mut(), + UpdateStatus::Enqueued(u) => u.content_path_mut(), + } + } } impl From for UpdateStatus { From 40ced3ff8d1eb51358c0d134129866e7bfed7000 Mon Sep 17 00:00:00 2001 From: tamo Date: Thu, 6 May 2021 18:44:16 +0200 Subject: [PATCH 322/522] first working version --- .../src/index_controller/dump_actor/mod.rs | 29 ++++++++++-- .../src/index_controller/dump_actor/v1.rs | 4 +- .../src/index_controller/dump_actor/v2.rs | 14 ++++-- .../src/index_controller/index_actor/mod.rs | 2 +- .../update_actor/update_store.rs | 46 +++++++++++++------ 5 files changed, 68 insertions(+), 27 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index f57c27c59..eb2bc4684 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -59,10 +59,11 @@ impl DumpVersion { size: usize, dump_path: &Path, index_path: &Path, + primary_key: Option<&str>, ) -> anyhow::Result<()> { match self { - Self::V1 => v1::import_index(size, dump_path, index_path), - Self::V2 => v2::import_index(size, dump_path, index_path), + Self::V1 => v1::import_index(size, dump_path, index_path, primary_key), + Self::V2 => v2::import_index(size, dump_path, index_path, primary_key), } } } @@ -206,7 +207,26 @@ pub fn load_dump( // this cannot fail since we created all the missing uuid in the previous loop let uuid = uuid_resolver.get_uuid(idx.uid)?.unwrap(); let index_path = db_path.join(&format!("indexes/index-{}", uuid)); - // let update_path = db_path.join(&format!("updates/updates-{}", uuid)); // TODO: add the update db + // let update_path = db_path.join(&format!("updates")); + + info!("importing the updates"); + use crate::index_controller::update_actor::UpdateStore; + use std::io::BufRead; + + let update_path = db_path.join("updates"); + let options = EnvOpenOptions::new(); + // create an UpdateStore to import the updates + std::fs::create_dir_all(&update_path)?; + let (update_store, _) = UpdateStore::create(options, update_path)?; + let file = File::open(&dump_path.join("updates.jsonl"))?; + let reader = std::io::BufReader::new(file); + + let mut wtxn = update_store.env.write_txn()?; + for update in reader.lines() { + let update = serde_json::from_str(&update?)?; + update_store.register_raw_updates(&mut wtxn, update, uuid)?; + } + wtxn.commit()?; info!( "Importing dump from {} into {}...", @@ -215,11 +235,12 @@ pub fn load_dump( ); metadata .dump_version - .import_index(size, &dump_path, &index_path) + .import_index(size, &dump_path, &index_path, idx.meta.primary_key.as_ref().map(|s| s.as_ref())) .unwrap(); info!("Dump importation from {} succeed", dump_path.display()); } + info!("Dump importation from {} succeed", dump_path.display()); Ok(()) } diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs index f22120849..d20723e8c 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v1.rs @@ -78,7 +78,7 @@ fn import_settings(dir_path: &Path) -> anyhow::Result { } -pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { +pub fn import_index(size: usize, dump_path: &Path, index_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { info!("Importing a dump from an old version of meilisearch with dump version 1"); std::fs::create_dir_all(&index_path)?; @@ -102,7 +102,7 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: IndexDocumentsMethod::ReplaceDocuments, Some(reader), update_builder, - None, + primary_key, )?; // at this point we should handle the updates, but since the update logic is not handled in diff --git a/meilisearch-http/src/index_controller/dump_actor/v2.rs b/meilisearch-http/src/index_controller/dump_actor/v2.rs index 5c5e5fb2d..301268233 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v2.rs @@ -1,5 +1,5 @@ use heed::EnvOpenOptions; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::{update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}}; use crate::index::Index; use crate::index_controller::Settings; use std::{fs::File, path::Path, sync::Arc}; @@ -14,7 +14,7 @@ fn import_settings(dir_path: &Path) -> anyhow::Result { Ok(metadata) } -pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { +pub fn import_index(size: usize, dump_path: &Path, index_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { std::fs::create_dir_all(&index_path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); @@ -26,17 +26,21 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path) -> anyhow: let update_builder = UpdateBuilder::new(0); index.update_settings(&settings, update_builder)?; + // import the documents in the index let update_builder = UpdateBuilder::new(1); let file = File::open(&dump_path.join("documents.jsonl"))?; let reader = std::io::BufReader::new(file); - index.update_documents( + // TODO: TAMO: currently we ignore any error caused by the importation of the documents because + // if there is no documents nor primary key it'll throw an anyhow error, but we must remove + // this before the merge on main + let _ = index.update_documents( UpdateFormat::JsonStream, IndexDocumentsMethod::ReplaceDocuments, Some(reader), update_builder, - None, - )?; + primary_key, + ); // the last step: we extract the original milli::Index and close it Arc::try_unwrap(index.0) diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index cf6a81223..3b92b1078 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -31,7 +31,7 @@ pub type IndexResult = std::result::Result; pub struct IndexMeta { created_at: DateTime, pub updated_at: DateTime, - primary_key: Option, + pub primary_key: Option, } impl IndexMeta { diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 524fefe84..745311f05 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -250,21 +250,31 @@ impl UpdateStore { .get(txn, &NextIdKey::Global)? .map(U64::get) .unwrap_or_default(); + + self.next_update_id + .put(txn, &NextIdKey::Global, &BEU64::new(global_id + 1))?; + + let update_id = self.next_update_id_raw(txn, index_uuid)?; + + Ok((global_id, update_id)) + } + + /// Returns the next next update id for a given `index_uuid` without + /// incrementing the global update id. This is useful for the dumps. + fn next_update_id_raw(&self, txn: &mut heed::RwTxn, index_uuid: Uuid) -> heed::Result { let update_id = self .next_update_id .get(txn, &NextIdKey::Index(index_uuid))? .map(U64::get) .unwrap_or_default(); - self.next_update_id - .put(txn, &NextIdKey::Global, &BEU64::new(global_id + 1))?; self.next_update_id.put( txn, &NextIdKey::Index(index_uuid), &BEU64::new(update_id + 1), )?; - Ok((global_id, update_id)) + Ok(update_id) } /// Registers the update content in the pending store and the meta @@ -291,17 +301,27 @@ impl UpdateStore { Ok(meta) } - /// Push already processed updates in the UpdateStore. This is useful for the dumps - pub fn register_already_processed_update ( + /// Push already processed update in the UpdateStore without triggering the notification + /// process. This is useful for the dumps. + pub fn register_raw_updates ( &self, - result: UpdateStatus, + wtxn: &mut heed::RwTxn, + update: UpdateStatus, index_uuid: Uuid, ) -> heed::Result<()> { - // TODO: TAMO: load already processed updates - let mut wtxn = self.env.write_txn()?; - let (_global_id, update_id) = self.next_update_id(&mut wtxn, index_uuid)?; - self.updates.remap_key_type::().put(&mut wtxn, &(index_uuid, update_id), &result)?; - wtxn.commit() + // TODO: TAMO: since I don't want to store anything I currently generate a new global ID + // everytime I encounter an enqueued update, can we do better? + match update { + UpdateStatus::Enqueued(enqueued) => { + let (global_id, update_id) = self.next_update_id(wtxn, index_uuid)?; + self.pending_queue.remap_key_type::().put(wtxn, &(global_id, index_uuid, update_id), &enqueued)?; + } + _ => { + let update_id = self.next_update_id_raw(wtxn, index_uuid)?; + self.updates.remap_key_type::().put(wtxn, &(index_uuid, update_id), &update)?; + } + } + Ok(()) } /// Executes the user provided function on the next pending update (the one with the lowest id). @@ -542,9 +562,6 @@ impl UpdateStore { } } - // TODO: TAMO: the updates - // already processed updates seems to works, but I've not tried with currently running updates - let update_files_path = path.join("update_files"); create_dir_all(&update_files_path)?; @@ -561,7 +578,6 @@ impl UpdateStore { } } - // Perform the dump of each index concurently. Only a third of the capabilities of // the index actor at a time not to put too much pressure on the index actor let path = &path; From ef438852cd0579878607264983aef4ccd85e2a59 Mon Sep 17 00:00:00 2001 From: tamo Date: Thu, 6 May 2021 18:47:56 +0200 Subject: [PATCH 323/522] fix the v1 --- meilisearch-http/src/index_controller/dump_actor/v1.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs index d20723e8c..89c5b24c0 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v1.rs @@ -51,7 +51,7 @@ impl From for index_controller::Settings { warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); Some(String::from("words")) } - "attribute" | "exactness" => { + "exactness" => { error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); None } @@ -97,13 +97,14 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path, primary_ke let file = File::open(&dump_path.join("documents.jsonl"))?; let reader = std::io::BufReader::new(file); - index.update_documents( + // TODO: TAMO: waiting for milli. We should use the result + let _ = index.update_documents( UpdateFormat::JsonStream, IndexDocumentsMethod::ReplaceDocuments, Some(reader), update_builder, primary_key, - )?; + ); // at this point we should handle the updates, but since the update logic is not handled in // meilisearch we are just going to ignore this part From d7679904240fa5771c995cdd9ece72d3939c467c Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 10 May 2021 20:20:36 +0200 Subject: [PATCH 324/522] fix the import of the updates in the dump --- meilisearch-http/src/index/mod.rs | 9 --- meilisearch-http/src/index/updates.rs | 6 +- .../src/index_controller/dump_actor/mod.rs | 76 +++++++++++------- .../src/index_controller/dump_actor/v1.rs | 6 +- .../update_actor/update_store.rs | 77 ++++++++++++++----- .../src/index_controller/updates.rs | 2 +- .../tests/settings/get_settings.rs | 4 +- 7 files changed, 116 insertions(+), 64 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index b0c145001..ceaa6103e 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -35,15 +35,6 @@ where Deserialize::deserialize(deserializer).map(Some) } -pub fn deserialize_wildcard<'de, I, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, - I: IntoIterator + Deserialize<'de> + Clone, -{ - Ok( as Deserialize>::deserialize(deserializer)? - .map(|item: I| (!item.clone().into_iter().any(|s| s == "*")).then(|| item))) -} - impl Index { pub fn settings(&self) -> anyhow::Result> { let txn = self.read_txn()?; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 75d0dc3e6..67edc15d0 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -8,7 +8,7 @@ use log::info; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; -use super::{deserialize_some, deserialize_wildcard, Index}; +use super::{deserialize_some, Index}; use crate::index_controller::UpdateResult; @@ -23,14 +23,14 @@ pub struct Unchecked; pub struct Settings { #[serde( default, - deserialize_with = "deserialize_wildcard", + deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none" )] pub displayed_attributes: Option>>, #[serde( default, - deserialize_with = "deserialize_wildcard", + deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none" )] pub searchable_attributes: Option>>, diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index eb2bc4684..f79cd839b 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,33 +1,29 @@ +mod actor; +mod handle_impl; +mod message; mod v1; mod v2; -mod handle_impl; -mod actor; -mod message; -use std::{ - fs::File, - path::Path, - sync::Arc, -}; +use std::{fs::File, path::Path, sync::Arc}; -#[cfg(test)] -use mockall::automock; use anyhow::bail; -use thiserror::Error; use heed::EnvOpenOptions; use log::{error, info}; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +#[cfg(test)] +use mockall::automock; use serde::{Deserialize, Serialize}; use serde_json::json; use tempfile::TempDir; +use thiserror::Error; use super::IndexMetadata; use crate::helpers::compression; use crate::index::Index; -use crate::index_controller::uuid_resolver; +use crate::index_controller::{uuid_resolver, UpdateStatus}; -pub use handle_impl::*; pub use actor::DumpActor; +pub use handle_impl::*; pub use message::DumpMsg; pub type DumpResult = std::result::Result; @@ -80,7 +76,6 @@ pub trait DumpActorHandle { async fn dump_info(&self, uid: String) -> DumpResult; } - #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Metadata { @@ -158,7 +153,6 @@ impl DumpInfo { } } - pub fn load_dump( db_path: impl AsRef, dump_path: impl AsRef, @@ -209,6 +203,22 @@ pub fn load_dump( let index_path = db_path.join(&format!("indexes/index-{}", uuid)); // let update_path = db_path.join(&format!("updates")); + info!( + "Importing dump from {} into {}...", + dump_path.display(), + index_path.display() + ); + metadata + .dump_version + .import_index( + size, + &dump_path, + &index_path, + idx.meta.primary_key.as_ref().map(|s| s.as_ref()), + ) + .unwrap(); + info!("Dump importation from {} succeed", dump_path.display()); + info!("importing the updates"); use crate::index_controller::update_actor::UpdateStore; use std::io::BufRead; @@ -217,29 +227,39 @@ pub fn load_dump( let options = EnvOpenOptions::new(); // create an UpdateStore to import the updates std::fs::create_dir_all(&update_path)?; - let (update_store, _) = UpdateStore::create(options, update_path)?; + let (update_store, _) = UpdateStore::create(options, &update_path)?; let file = File::open(&dump_path.join("updates.jsonl"))?; let reader = std::io::BufReader::new(file); let mut wtxn = update_store.env.write_txn()?; for update in reader.lines() { - let update = serde_json::from_str(&update?)?; + let mut update: UpdateStatus = serde_json::from_str(&update?)?; + if let Some(path) = update.content_path_mut() { + *path = update_path.join("update_files").join(&path).into(); + } update_store.register_raw_updates(&mut wtxn, update, uuid)?; } wtxn.commit()?; - - info!( - "Importing dump from {} into {}...", - dump_path.display(), - index_path.display() - ); - metadata - .dump_version - .import_index(size, &dump_path, &index_path, idx.meta.primary_key.as_ref().map(|s| s.as_ref())) - .unwrap(); - info!("Dump importation from {} succeed", dump_path.display()); } + // finally we can move all the unprocessed update file into our new DB + let update_path = tmp_dir_path.join("update_files"); + let files: Vec<_> = std::fs::read_dir(&db_path.join("updates"))? + .map(|file| file.unwrap().path()) + .collect(); + let db_update_path = db_path.join("updates/update_files"); + eprintln!("path {:?} exists: {:?}", update_path, update_path.exists()); + eprintln!( + "path {:?} exists: {:?}", + db_update_path, + db_update_path.exists() + ); + let _ = std::fs::remove_dir_all(db_update_path); + std::fs::rename( + tmp_dir_path.join("update_files"), + db_path.join("updates/update_files"), + ) + .unwrap(); info!("Dump importation from {} succeed", dump_path.display()); Ok(()) diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs index 89c5b24c0..92f8bf712 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v1.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use log::warn; use serde::{Deserialize, Serialize}; use crate::index_controller; -use crate::index::{deserialize_wildcard, deserialize_some}; +use crate::index::deserialize_some; use super::*; /// This is the settings used in the last version of meilisearch exporting dump in V1 @@ -14,9 +14,9 @@ struct Settings { pub ranking_rules: Option>>, #[serde(default, deserialize_with = "deserialize_some")] pub distinct_attribute: Option>, - #[serde(default, deserialize_with = "deserialize_wildcard")] + #[serde(default, deserialize_with = "deserialize_some")] pub searchable_attributes: Option>>, - #[serde(default, deserialize_with = "deserialize_wildcard")] + #[serde(default, deserialize_with = "deserialize_some")] pub displayed_attributes: Option>>, #[serde(default, deserialize_with = "deserialize_some")] pub stop_words: Option>>, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 745311f05..07dfdf273 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -1,13 +1,14 @@ -use std::{borrow::Cow, path::PathBuf}; use std::collections::{BTreeMap, HashSet}; use std::convert::TryInto; use std::fs::{copy, create_dir_all, remove_file, File}; use std::mem::size_of; use std::path::Path; use std::sync::Arc; +use std::{borrow::Cow, path::PathBuf}; use anyhow::Context; use arc_swap::ArcSwap; +use futures::StreamExt; use heed::types::{ByteSlice, OwnedType, SerdeJson}; use heed::zerocopy::U64; use heed::{BytesDecode, BytesEncode, CompactionOption, Database, Env, EnvOpenOptions}; @@ -16,11 +17,10 @@ use parking_lot::{Mutex, MutexGuard}; use tokio::runtime::Handle; use tokio::sync::mpsc; use uuid::Uuid; -use futures::StreamExt; use super::UpdateMeta; use crate::helpers::EnvSizer; -use crate::index_controller::{IndexActorHandle, updates::*, index_actor::CONCURRENT_INDEX_MSG}; +use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle}; #[allow(clippy::upper_case_acronyms)] type BEU64 = U64; @@ -180,7 +180,10 @@ pub struct UpdateStore { } impl UpdateStore { - pub fn create(mut options: EnvOpenOptions, path: impl AsRef) -> anyhow::Result<(Self, mpsc::Receiver<()>)> { + pub fn create( + mut options: EnvOpenOptions, + path: impl AsRef, + ) -> anyhow::Result<(Self, mpsc::Receiver<()>)> { options.max_dbs(5); let env = options.open(path)?; @@ -194,7 +197,17 @@ impl UpdateStore { // Send a first notification to trigger the process. let _ = notification_sender.send(()); - Ok((Self { env, pending_queue, next_update_id, updates, state, notification_sender }, notification_receiver)) + Ok(( + Self { + env, + pending_queue, + next_update_id, + updates, + state, + notification_sender, + }, + notification_receiver, + )) } pub fn open( @@ -208,7 +221,8 @@ impl UpdateStore { // Init update loop to perform any pending updates at launch. // Since we just launched the update store, and we still own the receiving end of the // channel, this call is guaranteed to succeed. - update_store.notification_sender + update_store + .notification_sender .try_send(()) .expect("Failed to init update store"); @@ -303,22 +317,28 @@ impl UpdateStore { /// Push already processed update in the UpdateStore without triggering the notification /// process. This is useful for the dumps. - pub fn register_raw_updates ( + pub fn register_raw_updates( &self, wtxn: &mut heed::RwTxn, update: UpdateStatus, index_uuid: Uuid, ) -> heed::Result<()> { - // TODO: TAMO: since I don't want to store anything I currently generate a new global ID - // everytime I encounter an enqueued update, can we do better? match update { UpdateStatus::Enqueued(enqueued) => { - let (global_id, update_id) = self.next_update_id(wtxn, index_uuid)?; - self.pending_queue.remap_key_type::().put(wtxn, &(global_id, index_uuid, update_id), &enqueued)?; + let (global_id, _update_id) = self.next_update_id(wtxn, index_uuid)?; + self.pending_queue.remap_key_type::().put( + wtxn, + &(global_id, index_uuid, enqueued.id()), + &enqueued, + )?; } _ => { - let update_id = self.next_update_id_raw(wtxn, index_uuid)?; - self.updates.remap_key_type::().put(wtxn, &(index_uuid, update_id), &update)?; + let _update_id = self.next_update_id_raw(wtxn, index_uuid)?; + self.updates.remap_key_type::().put( + wtxn, + &(index_uuid, update.id()), + &update, + )?; } } Ok(()) @@ -544,20 +564,39 @@ impl UpdateStore { &self, uuids: &HashSet<(String, Uuid)>, path: PathBuf, - handle: impl IndexActorHandle - ) -> anyhow::Result<()> { + handle: impl IndexActorHandle, + ) -> anyhow::Result<()> { use std::io::prelude::*; let state_lock = self.state.write(); state_lock.swap(State::Dumping); let txn = self.env.write_txn()?; - for (uid, uuid) in uuids.iter() { - let file = File::create(path.join(uid).join("updates.jsonl"))?; + for (index_uid, index_uuid) in uuids.iter() { + let file = File::create(path.join(index_uid).join("updates.jsonl"))?; let mut file = std::io::BufWriter::new(file); - for update in &self.list(*uuid)? { - serde_json::to_writer(&mut file, update)?; + let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); + for entry in pendings { + let ((_, uuid, _), pending) = entry?; + if &uuid == index_uuid { + let mut update: UpdateStatus = pending.decode()?.into(); + if let Some(path) = update.content_path_mut() { + *path = path.file_name().expect("update path can't be empty").into(); + } + serde_json::to_writer(&mut file, &update)?; + file.write_all(b"\n")?; + } + } + + let updates = self.updates.prefix_iter(&txn, index_uuid.as_bytes())?; + for entry in updates { + let (_, update) = entry?; + let mut update = update.clone(); + if let Some(path) = update.content_path_mut() { + *path = path.file_name().expect("update path can't be empty").into(); + } + serde_json::to_writer(&mut file, &update)?; file.write_all(b"\n")?; } } diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 6b5ef345d..31f0005f8 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -188,7 +188,7 @@ impl Failed { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] #[serde(tag = "status", rename_all = "camelCase")] pub enum UpdateStatus { Processing(Processing), diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 4230e19f8..ab688076d 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -73,7 +73,7 @@ async fn reset_all_settings() { let server = Server::new().await; let index = server.index("test"); index - .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"], "stopWords": ["the"] })) + .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"], "stopWords": ["the"], "attributesForFaceting": { "toto": "string" } })) .await; index.wait_update_id(0).await; let (response, code) = index.settings().await; @@ -81,6 +81,7 @@ async fn reset_all_settings() { assert_eq!(response["displayedAttributes"], json!(["foo"])); assert_eq!(response["searchableAttributes"], json!(["bar"])); assert_eq!(response["stopWords"], json!(["the"])); + assert_eq!(response["attributesForFaceting"], json!({"toto": "string"})); index.delete_settings().await; index.wait_update_id(1).await; @@ -90,6 +91,7 @@ async fn reset_all_settings() { assert_eq!(response["displayedAttributes"], json!(["*"])); assert_eq!(response["searchableAttributes"], json!(["*"])); assert_eq!(response["stopWords"], json!([])); + assert_eq!(response["attributesForFaceting"], json!({})); } #[actix_rt::test] From 7d748fa3841287131b6252479bb53d336a611ed8 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 10 May 2021 20:48:06 +0200 Subject: [PATCH 325/522] integrate the new Settings in the dumps --- meilisearch-http/src/index/mod.rs | 2 +- meilisearch-http/src/index/updates.rs | 9 +++++++-- .../src/index_controller/dump_actor/mod.rs | 3 --- .../src/index_controller/dump_actor/v1.rs | 11 ++++++----- .../src/index_controller/dump_actor/v2.rs | 4 ++-- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index ceaa6103e..d3f30bf2e 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; use crate::helpers::EnvSizer; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; -pub use updates::{Facets, Settings, Checked, Unchecked, UpdateResult}; +pub use updates::{Facets, Settings, Checked, Unchecked}; use serde::{de::Deserializer, Deserialize}; mod search; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 67edc15d0..0762c8550 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -8,9 +8,10 @@ use log::info; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; -use super::{deserialize_some, Index}; use crate::index_controller::UpdateResult; +use super::{deserialize_some, Index}; + #[derive(Clone, Default, Debug)] pub struct Checked; @@ -35,7 +36,11 @@ pub struct Settings { )] pub searchable_attributes: Option>>, - #[serde(default)] + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] pub attributes_for_faceting: Option>>, #[serde( diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index f79cd839b..a8409f623 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -244,9 +244,6 @@ pub fn load_dump( // finally we can move all the unprocessed update file into our new DB let update_path = tmp_dir_path.join("update_files"); - let files: Vec<_> = std::fs::read_dir(&db_path.join("updates"))? - .map(|file| file.unwrap().path()) - .collect(); let db_update_path = db_path.join("updates/update_files"); eprintln!("path {:?} exists: {:?}", update_path, update_path.exists()); eprintln!( diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs index 92f8bf712..33fab6930 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v1.rs @@ -1,8 +1,8 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::{collections::{BTreeMap, BTreeSet}, marker::PhantomData}; use log::warn; use serde::{Deserialize, Serialize}; -use crate::index_controller; +use crate::{index::Unchecked, index_controller}; use crate::index::deserialize_some; use super::*; @@ -27,7 +27,7 @@ struct Settings { } /// we need to **always** be able to convert the old settings to the settings currently being used -impl From for index_controller::Settings { +impl From for index_controller::Settings { fn from(settings: Settings) -> Self { if settings.synonyms.flatten().is_some() { error!("`synonyms` are not yet implemented and thus will be ignored"); @@ -63,6 +63,7 @@ impl From for index_controller::Settings { }).collect())), // we need to convert the old `Vec` into a `BTreeSet` stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())), + _kind: PhantomData, } } } @@ -89,9 +90,9 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path, primary_ke // extract `settings.json` file and import content let settings = import_settings(&dump_path)?; - let settings: index_controller::Settings = settings.into(); + let settings: index_controller::Settings = settings.into(); let update_builder = UpdateBuilder::new(0); - index.update_settings(&settings, update_builder)?; + index.update_settings(&settings.check(), update_builder)?; let update_builder = UpdateBuilder::new(1); let file = File::open(&dump_path.join("documents.jsonl"))?; diff --git a/meilisearch-http/src/index_controller/dump_actor/v2.rs b/meilisearch-http/src/index_controller/dump_actor/v2.rs index 301268233..d8f43fc58 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v2.rs @@ -1,11 +1,11 @@ use heed::EnvOpenOptions; use milli::{update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}}; -use crate::index::Index; +use crate::index::{Checked, Index}; use crate::index_controller::Settings; use std::{fs::File, path::Path, sync::Arc}; /// Extract Settings from `settings.json` file present at provided `dir_path` -fn import_settings(dir_path: &Path) -> anyhow::Result { +fn import_settings(dir_path: &Path) -> anyhow::Result> { let path = dir_path.join("settings.json"); let file = File::open(path)?; let reader = std::io::BufReader::new(file); From 8b7735c20a1779059a47ab58242b9f67320f0e46 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 11 May 2021 00:20:55 +0200 Subject: [PATCH 326/522] move the import of the updates in the v2 and ignore the v1 for now --- .../src/index_controller/dump_actor/mod.rs | 39 +++++-------------- .../src/index_controller/dump_actor/v1.rs | 3 +- .../src/index_controller/dump_actor/v2.rs | 32 ++++++++++++++- 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index a8409f623..d416d7d92 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -16,11 +16,12 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use tempfile::TempDir; use thiserror::Error; +use uuid::Uuid; use super::IndexMetadata; use crate::helpers::compression; use crate::index::Index; -use crate::index_controller::{uuid_resolver, UpdateStatus}; +use crate::index_controller::uuid_resolver; pub use actor::DumpActor; pub use handle_impl::*; @@ -53,13 +54,14 @@ impl DumpVersion { pub fn import_index( self, size: usize, + uuid: Uuid, dump_path: &Path, - index_path: &Path, + db_path: &Path, primary_key: Option<&str>, ) -> anyhow::Result<()> { match self { - Self::V1 => v1::import_index(size, dump_path, index_path, primary_key), - Self::V2 => v2::import_index(size, dump_path, index_path, primary_key), + Self::V1 => v1::import_index(size, uuid, dump_path, db_path, primary_key), + Self::V2 => v2::import_index(size, uuid, dump_path, db_path, primary_key), } } } @@ -200,46 +202,23 @@ pub fn load_dump( let dump_path = tmp_dir_path.join(&idx.uid); // this cannot fail since we created all the missing uuid in the previous loop let uuid = uuid_resolver.get_uuid(idx.uid)?.unwrap(); - let index_path = db_path.join(&format!("indexes/index-{}", uuid)); - // let update_path = db_path.join(&format!("updates")); info!( "Importing dump from {} into {}...", dump_path.display(), - index_path.display() + db_path.display() ); metadata .dump_version .import_index( size, + uuid, &dump_path, - &index_path, + &db_path, idx.meta.primary_key.as_ref().map(|s| s.as_ref()), ) .unwrap(); info!("Dump importation from {} succeed", dump_path.display()); - - info!("importing the updates"); - use crate::index_controller::update_actor::UpdateStore; - use std::io::BufRead; - - let update_path = db_path.join("updates"); - let options = EnvOpenOptions::new(); - // create an UpdateStore to import the updates - std::fs::create_dir_all(&update_path)?; - let (update_store, _) = UpdateStore::create(options, &update_path)?; - let file = File::open(&dump_path.join("updates.jsonl"))?; - let reader = std::io::BufReader::new(file); - - let mut wtxn = update_store.env.write_txn()?; - for update in reader.lines() { - let mut update: UpdateStatus = serde_json::from_str(&update?)?; - if let Some(path) = update.content_path_mut() { - *path = update_path.join("update_files").join(&path).into(); - } - update_store.register_raw_updates(&mut wtxn, update, uuid)?; - } - wtxn.commit()?; } // finally we can move all the unprocessed update file into our new DB diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs index 33fab6930..fad48dd8f 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v1.rs @@ -79,7 +79,8 @@ fn import_settings(dir_path: &Path) -> anyhow::Result { } -pub fn import_index(size: usize, dump_path: &Path, index_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { +pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { + let index_path = db_path.join(&format!("indexes/index-{}", uuid)); info!("Importing a dump from an old version of meilisearch with dump version 1"); std::fs::create_dir_all(&index_path)?; diff --git a/meilisearch-http/src/index_controller/dump_actor/v2.rs b/meilisearch-http/src/index_controller/dump_actor/v2.rs index d8f43fc58..969442296 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v2.rs @@ -1,4 +1,8 @@ use heed::EnvOpenOptions; +use log::info; +use uuid::Uuid; +use crate::index_controller::{UpdateStatus, update_actor::UpdateStore}; +use std::io::BufRead; use milli::{update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}}; use crate::index::{Checked, Index}; use crate::index_controller::Settings; @@ -14,13 +18,15 @@ fn import_settings(dir_path: &Path) -> anyhow::Result> { Ok(metadata) } -pub fn import_index(size: usize, dump_path: &Path, index_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { +pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { + let index_path = db_path.join(&format!("indexes/index-{}", uuid)); std::fs::create_dir_all(&index_path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); let index = milli::Index::new(options, index_path)?; let index = Index(Arc::new(index)); + info!("importing the settings..."); // extract `settings.json` file and import content let settings = import_settings(&dump_path)?; let update_builder = UpdateBuilder::new(0); @@ -31,6 +37,7 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path, primary_ke let file = File::open(&dump_path.join("documents.jsonl"))?; let reader = std::io::BufReader::new(file); + info!("importing the documents..."); // TODO: TAMO: currently we ignore any error caused by the importation of the documents because // if there is no documents nor primary key it'll throw an anyhow error, but we must remove // this before the merge on main @@ -49,6 +56,27 @@ pub fn import_index(size: usize, dump_path: &Path, index_path: &Path, primary_ke .prepare_for_closing() .wait(); - Ok(()) + info!("importing the updates..."); + import_updates(uuid, dump_path, db_path) } +fn import_updates(uuid: Uuid, dump_path: &Path, db_path: &Path) -> anyhow::Result<()> { + let update_path = db_path.join("updates"); + let options = EnvOpenOptions::new(); + // create an UpdateStore to import the updates + std::fs::create_dir_all(&update_path)?; + let (update_store, _) = UpdateStore::create(options, &update_path)?; + let file = File::open(&dump_path.join("updates.jsonl"))?; + let reader = std::io::BufReader::new(file); + + let mut wtxn = update_store.env.write_txn()?; + for update in reader.lines() { + let mut update: UpdateStatus = serde_json::from_str(&update?)?; + if let Some(path) = update.content_path_mut() { + *path = update_path.join("update_files").join(&path).into(); + } + update_store.register_raw_updates(&mut wtxn, update, uuid)?; + } + wtxn.commit()?; + Ok(()) +} From 92a7c8cd176a7cbcf4ab7f20be3341505333a9d1 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 11 May 2021 00:27:22 +0200 Subject: [PATCH 327/522] make clippy happy --- meilisearch-http/src/index_controller/dump_actor/v1.rs | 8 ++++---- meilisearch-http/src/index_controller/dump_actor/v2.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs index fad48dd8f..6844ea241 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v1.rs @@ -86,7 +86,7 @@ pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, p std::fs::create_dir_all(&index_path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options.clone(), index_path)?; + let index = milli::Index::new(options, index_path)?; let index = Index(Arc::new(index)); // extract `settings.json` file and import content @@ -108,9 +108,6 @@ pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, p primary_key, ); - // at this point we should handle the updates, but since the update logic is not handled in - // meilisearch we are just going to ignore this part - // the last step: we extract the original milli::Index and close it Arc::try_unwrap(index.0) .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") @@ -118,5 +115,8 @@ pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, p .prepare_for_closing() .wait(); + // at this point we should handle the import of the updates, but since the update logic is not handled in + // meilisearch we are just going to ignore this part + Ok(()) } diff --git a/meilisearch-http/src/index_controller/dump_actor/v2.rs b/meilisearch-http/src/index_controller/dump_actor/v2.rs index 969442296..4f39f88bf 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v2.rs @@ -73,7 +73,7 @@ fn import_updates(uuid: Uuid, dump_path: &Path, db_path: &Path) -> anyhow::Resul for update in reader.lines() { let mut update: UpdateStatus = serde_json::from_str(&update?)?; if let Some(path) = update.content_path_mut() { - *path = update_path.join("update_files").join(&path).into(); + *path = update_path.join("update_files").join(&path); } update_store.register_raw_updates(&mut wtxn, update, uuid)?; } From 384afb3455212f4cf375d23a5d088fd24296fc36 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 11 May 2021 11:47:04 +0200 Subject: [PATCH 328/522] fix the way we return the settings --- meilisearch-http/src/index/mod.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index d3f30bf2e..f26cc4283 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -41,13 +41,11 @@ impl Index { let displayed_attributes = self .displayed_fields(&txn)? - .map(|fields| fields.into_iter().map(String::from).collect()) - .unwrap_or_else(|| vec!["*".to_string()]); + .map(|fields| fields.into_iter().map(String::from).collect()); let searchable_attributes = self .searchable_fields(&txn)? - .map(|fields| fields.into_iter().map(String::from).collect()) - .unwrap_or_else(|| vec!["*".to_string()]); + .map(|fields| fields.into_iter().map(String::from).collect()); let faceted_attributes = self .faceted_fields(&txn)? @@ -71,8 +69,8 @@ impl Index { let distinct_attribute = self.distinct_attribute(&txn)?.map(String::from); Ok(Settings { - displayed_attributes: Some(Some(displayed_attributes)), - searchable_attributes: Some(Some(searchable_attributes)), + displayed_attributes: Some(displayed_attributes), + searchable_attributes: Some(searchable_attributes), attributes_for_faceting: Some(Some(faceted_attributes)), ranking_rules: Some(Some(criteria)), stop_words: Some(Some(stop_words)), From 9e798fea75bcd8f1cee4b98b014f61f49df28715 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 11 May 2021 13:03:47 +0200 Subject: [PATCH 329/522] fix the import of dump without unprocessing updates --- .../src/index_controller/dump_actor/mod.rs | 37 ++++++++----------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index d416d7d92..7d2e5a951 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -208,34 +208,27 @@ pub fn load_dump( dump_path.display(), db_path.display() ); - metadata - .dump_version - .import_index( - size, - uuid, - &dump_path, - &db_path, - idx.meta.primary_key.as_ref().map(|s| s.as_ref()), - ) - .unwrap(); + metadata.dump_version.import_index( + size, + uuid, + &dump_path, + &db_path, + idx.meta.primary_key.as_ref().map(|s| s.as_ref()), + )?; info!("Dump importation from {} succeed", dump_path.display()); } // finally we can move all the unprocessed update file into our new DB + // this directory may not exists let update_path = tmp_dir_path.join("update_files"); let db_update_path = db_path.join("updates/update_files"); - eprintln!("path {:?} exists: {:?}", update_path, update_path.exists()); - eprintln!( - "path {:?} exists: {:?}", - db_update_path, - db_update_path.exists() - ); - let _ = std::fs::remove_dir_all(db_update_path); - std::fs::rename( - tmp_dir_path.join("update_files"), - db_path.join("updates/update_files"), - ) - .unwrap(); + if update_path.exists() { + let _ = std::fs::remove_dir_all(db_update_path); + std::fs::rename( + tmp_dir_path.join("update_files"), + db_path.join("updates/update_files"), + )?; + } info!("Dump importation from {} succeed", dump_path.display()); Ok(()) From c30b32e173548cd02b0cddf683ff6e41c415a65b Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 11 May 2021 13:21:36 +0200 Subject: [PATCH 330/522] add the criterion attribute when importing dumps from the v1 --- meilisearch-http/src/index_controller/dump_actor/v1.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs index 6844ea241..6f199193c 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v1.rs @@ -45,7 +45,7 @@ impl From for index_controller::Settings { // we need to convert the old `Vec` into a `BTreeSet` ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { match criterion.as_str() { - "words" | "typo" | "proximity" => Some(criterion), + "words" | "typo" | "proximity" | "attribute" => Some(criterion), s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), "wordsPosition" => { warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); From 1b671d4302e0effdcd70e0f2cf98c3121756fa81 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 11 May 2021 12:18:10 +0200 Subject: [PATCH 331/522] fix-snapshot --- .github/workflows/rust.yml | 1 + .../index_controller/update_actor/actor.rs | 23 +---------- .../update_actor/update_store.rs | 39 +++++++++++++++++-- meilisearch-http/tests/snapshot/mod.rs | 1 - 4 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ee23eb487..8020ead32 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,6 +1,7 @@ name: Rust on: + workflow_dispatch: pull_request: push: # trying and staging branches are for Bors config diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index e47edc5bc..b26ba4b8e 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -3,17 +3,15 @@ use std::io::SeekFrom; use std::path::{Path, PathBuf}; use std::sync::Arc; -use futures::StreamExt; use log::info; use oxidized_json_checker::JsonChecker; use tokio::fs; use tokio::io::AsyncWriteExt; -use tokio::runtime::Handle; use tokio::sync::mpsc; use uuid::Uuid; use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore, UpdateStoreInfo}; -use crate::index_controller::index_actor::{IndexActorHandle, CONCURRENT_INDEX_MSG}; +use crate::index_controller::index_actor::{IndexActorHandle}; use crate::index_controller::{UpdateMeta, UpdateStatus}; pub struct UpdateActor { @@ -207,25 +205,8 @@ where async fn handle_snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); - tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - update_store.snapshot(&uuids, &path)?; - // Perform the snapshot of each index concurently. Only a third of the capabilities of - // the index actor at a time not to put too much pressure on the index actor - let path = &path; - let handle = &index_handle; - - let mut stream = futures::stream::iter(uuids.iter()) - .map(|&uuid| handle.snapshot(uuid, path.clone())) - .buffer_unordered(CONCURRENT_INDEX_MSG / 3); - - Handle::current().block_on(async { - while let Some(res) = stream.next().await { - res?; - } - Ok(()) - }) - }) + tokio::task::spawn_blocking(move || update_store.snapshot(&uuids, &path, index_handle)) .await .map_err(|e| UpdateError::Error(e.into()))? .map_err(|e| UpdateError::Error(e.into()))?; diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 6a916af33..4e7acc7cf 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use anyhow::Context; use arc_swap::ArcSwap; +use futures::StreamExt; use heed::types::{ByteSlice, OwnedType, SerdeJson}; use heed::zerocopy::U64; use heed::{BytesDecode, BytesEncode, CompactionOption, Database, Env, EnvOpenOptions}; @@ -17,8 +18,11 @@ use tokio::sync::mpsc; use uuid::Uuid; use super::UpdateMeta; -use crate::helpers::EnvSizer; -use crate::index_controller::{IndexActorHandle, updates::*}; +use crate::index_controller::{updates::*, IndexActorHandle}; +use crate::{ + helpers::EnvSizer, + index_controller::index_actor::{IndexResult, CONCURRENT_INDEX_MSG}, +}; #[allow(clippy::upper_case_acronyms)] type BEU64 = U64; @@ -202,7 +206,14 @@ impl UpdateStore { .try_send(()) .expect("Failed to init update store"); - let update_store = Arc::new(UpdateStore { env, pending_queue, next_update_id, updates, state, notification_sender }); + let update_store = Arc::new(UpdateStore { + env, + pending_queue, + next_update_id, + updates, + state, + notification_sender, + }); // We need a weak reference so we can take ownership on the arc later when we // want to close the index. @@ -464,7 +475,12 @@ impl UpdateStore { Ok(()) } - pub fn snapshot(&self, uuids: &HashSet, path: impl AsRef) -> anyhow::Result<()> { + pub fn snapshot( + &self, + uuids: &HashSet, + path: impl AsRef, + handle: impl IndexActorHandle + Clone, + ) -> anyhow::Result<()> { let state_lock = self.state.write(); state_lock.swap(State::Snapshoting); @@ -496,6 +512,21 @@ impl UpdateStore { } } + let path = &path.as_ref().to_path_buf(); + let handle = &handle; + // Perform the snapshot of each index concurently. Only a third of the capabilities of + // the index actor at a time not to put too much pressure on the index actor + let mut stream = futures::stream::iter(uuids.iter()) + .map(move |uuid| handle.snapshot(*uuid, path.clone())) + .buffer_unordered(CONCURRENT_INDEX_MSG / 3); + + Handle::current().block_on(async { + while let Some(res) = stream.next().await { + res?; + } + Ok(()) as IndexResult<()> + })?; + Ok(()) } diff --git a/meilisearch-http/tests/snapshot/mod.rs b/meilisearch-http/tests/snapshot/mod.rs index caed293e6..b5602c508 100644 --- a/meilisearch-http/tests/snapshot/mod.rs +++ b/meilisearch-http/tests/snapshot/mod.rs @@ -7,7 +7,6 @@ use tokio::time::sleep; use meilisearch_http::Opt; -#[ignore] #[actix_rt::test] async fn perform_snapshot() { let temp = tempfile::tempdir_in(".").unwrap(); From 6d837e3e07a68ba561e25fb2986cde803b889c29 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 11 May 2021 17:34:34 +0200 Subject: [PATCH 332/522] the route to create a dump must return a 202 --- meilisearch-http/src/routes/dump.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index e6be4ca93..47c081e6f 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -17,7 +17,7 @@ async fn create_dump( ) -> Result { let res = data.create_dump().await?; - Ok(HttpResponse::Ok().json(res)) + Ok(HttpResponse::Accepted().json(res)) } #[derive(Debug, Serialize)] From 295f496e8a5b75ba27cb6f921f43eaea7a484fb9 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 12 May 2021 16:21:37 +0200 Subject: [PATCH 333/522] atomic index dump load --- meilisearch-http/src/index/updates.rs | 69 +++++++++++-------- .../src/index_controller/dump_actor/v2.rs | 19 +++-- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 0762c8550..b9c772ee2 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -132,17 +132,30 @@ impl Index { content: Option, update_builder: UpdateBuilder, primary_key: Option<&str>, + ) -> anyhow::Result { + let mut txn = self.write_txn()?; + let result = self.update_documents_txn(&mut txn, format, method, content, update_builder, primary_key)?; + txn.commit()?; + Ok(result) + } + + pub fn update_documents_txn<'a, 'b>( + &'a self, + txn: &mut heed::RwTxn<'a, 'b>, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: Option, + update_builder: UpdateBuilder, + primary_key: Option<&str>, ) -> anyhow::Result { info!("performing document addition"); - // We must use the write transaction of the update here. - let mut wtxn = self.write_txn()?; // Set the primary key if not set already, ignore if already set. - if let (None, Some(ref primary_key)) = (self.primary_key(&wtxn)?, primary_key) { - self.put_primary_key(&mut wtxn, primary_key)?; + if let (None, Some(ref primary_key)) = (self.primary_key(txn)?, primary_key) { + self.put_primary_key(txn, primary_key)?; } - let mut builder = update_builder.index_documents(&mut wtxn, self); + let mut builder = update_builder.index_documents(txn, self); builder.update_format(format); builder.index_documents_method(method); @@ -150,19 +163,15 @@ impl Index { |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); let gzipped = false; - let result = match content { - Some(content) if gzipped => builder.execute(GzDecoder::new(content), indexing_callback), - Some(content) => builder.execute(content, indexing_callback), - None => builder.execute(std::io::empty(), indexing_callback), + let addition = match content { + Some(content) if gzipped => builder.execute(GzDecoder::new(content), indexing_callback)?, + Some(content) => builder.execute(content, indexing_callback)?, + None => builder.execute(std::io::empty(), indexing_callback)?, }; - info!("document addition done: {:?}", result); + info!("document addition done: {:?}", addition); - result.and_then(|addition_result| { - wtxn.commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into) - }) + Ok(UpdateResult::DocumentsAddition(addition)) } pub fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { @@ -179,14 +188,14 @@ impl Index { } } - pub fn update_settings( - &self, + pub fn update_settings_txn<'a, 'b>( + &'a self, + txn: &mut heed::RwTxn<'a, 'b>, settings: &Settings, update_builder: UpdateBuilder, ) -> anyhow::Result { // We must use the write transaction of the update here. - let mut wtxn = self.write_txn()?; - let mut builder = update_builder.settings(&mut wtxn, self); + let mut builder = update_builder.settings(txn, self); if let Some(ref names) = settings.searchable_attributes { match names { @@ -228,16 +237,20 @@ impl Index { } } - let result = builder - .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step))?; - match result { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e), - } + Ok(UpdateResult::Other) + } + + pub fn update_settings( + &self, + settings: &Settings, + update_builder: UpdateBuilder, + ) -> anyhow::Result { + let mut txn = self.write_txn()?; + let result = self.update_settings_txn(&mut txn, settings, update_builder)?; + txn.commit()?; + Ok(result) } pub fn delete_documents( diff --git a/meilisearch-http/src/index_controller/dump_actor/v2.rs b/meilisearch-http/src/index_controller/dump_actor/v2.rs index 4f39f88bf..eeda78e8a 100644 --- a/meilisearch-http/src/index_controller/dump_actor/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/v2.rs @@ -1,7 +1,7 @@ use heed::EnvOpenOptions; use log::info; use uuid::Uuid; -use crate::index_controller::{UpdateStatus, update_actor::UpdateStore}; +use crate::{index::Unchecked, index_controller::{UpdateStatus, update_actor::UpdateStore}}; use std::io::BufRead; use milli::{update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}}; use crate::index::{Checked, Index}; @@ -13,9 +13,11 @@ fn import_settings(dir_path: &Path) -> anyhow::Result> { let path = dir_path.join("settings.json"); let file = File::open(path)?; let reader = std::io::BufReader::new(file); - let metadata = serde_json::from_reader(reader)?; + let metadata: Settings = serde_json::from_reader(reader)?; - Ok(metadata) + println!("Meta: {:?}", metadata); + + Ok(metadata.check()) } pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { @@ -26,11 +28,13 @@ pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, p let index = milli::Index::new(options, index_path)?; let index = Index(Arc::new(index)); + let mut txn = index.write_txn()?; + info!("importing the settings..."); // extract `settings.json` file and import content let settings = import_settings(&dump_path)?; let update_builder = UpdateBuilder::new(0); - index.update_settings(&settings, update_builder)?; + index.update_settings_txn(&mut txn, &settings, update_builder)?; // import the documents in the index let update_builder = UpdateBuilder::new(1); @@ -41,13 +45,16 @@ pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, p // TODO: TAMO: currently we ignore any error caused by the importation of the documents because // if there is no documents nor primary key it'll throw an anyhow error, but we must remove // this before the merge on main - let _ = index.update_documents( + index.update_documents_txn( + &mut txn, UpdateFormat::JsonStream, IndexDocumentsMethod::ReplaceDocuments, Some(reader), update_builder, primary_key, - ); + )?; + + txn.commit()?; // the last step: we extract the original milli::Index and close it Arc::try_unwrap(index.0) From e0e23636c64e6fce23d6dfacfbd089ef1833134e Mon Sep 17 00:00:00 2001 From: tamo Date: Wed, 12 May 2021 17:04:24 +0200 Subject: [PATCH 334/522] fix the serializer + reformat the file --- meilisearch-http/src/index/updates.rs | 35 ++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index b9c772ee2..0f4bf3589 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -1,17 +1,24 @@ use std::collections::{BTreeSet, HashMap}; use std::io; -use std::num::NonZeroUsize; use std::marker::PhantomData; +use std::num::NonZeroUsize; use flate2::read::GzDecoder; use log::info; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, Serializer}; use crate::index_controller::UpdateResult; use super::{deserialize_some, Index}; +fn serialize_with_wildcard(field: &Option>>, s: S) -> Result +where + S: Serializer, +{ + let wildcard = vec!["*".to_string()]; + s.serialize_some(&field.as_ref().map(|o| o.as_ref().unwrap_or(&wildcard))) +} #[derive(Clone, Default, Debug)] pub struct Checked; @@ -25,6 +32,7 @@ pub struct Settings { #[serde( default, deserialize_with = "deserialize_some", + serialize_with = "serialize_with_wildcard", skip_serializing_if = "Option::is_none" )] pub displayed_attributes: Option>>, @@ -32,6 +40,7 @@ pub struct Settings { #[serde( default, deserialize_with = "deserialize_some", + serialize_with = "serialize_with_wildcard", skip_serializing_if = "Option::is_none" )] pub searchable_attributes: Option>>, @@ -134,7 +143,14 @@ impl Index { primary_key: Option<&str>, ) -> anyhow::Result { let mut txn = self.write_txn()?; - let result = self.update_documents_txn(&mut txn, format, method, content, update_builder, primary_key)?; + let result = self.update_documents_txn( + &mut txn, + format, + method, + content, + update_builder, + primary_key, + )?; txn.commit()?; Ok(result) } @@ -164,7 +180,9 @@ impl Index { let gzipped = false; let addition = match content { - Some(content) if gzipped => builder.execute(GzDecoder::new(content), indexing_callback)?, + Some(content) if gzipped => { + builder.execute(GzDecoder::new(content), indexing_callback)? + } Some(content) => builder.execute(content, indexing_callback)?, None => builder.execute(std::io::empty(), indexing_callback)?, }; @@ -237,7 +255,9 @@ impl Index { } } - builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step))?; + builder.execute(|indexing_step, update_id| { + info!("update {}: {:?}", update_id, indexing_step) + })?; Ok(UpdateResult::Other) } @@ -299,7 +319,10 @@ mod test { let checked = settings.clone().check(); assert_eq!(settings.displayed_attributes, checked.displayed_attributes); - assert_eq!(settings.searchable_attributes, checked.searchable_attributes); + assert_eq!( + settings.searchable_attributes, + checked.searchable_attributes + ); // test wildcard // test no changes From 8a11c6c4291b1bcaa3ada62f15080d65e793a8b0 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 24 May 2021 12:35:46 +0200 Subject: [PATCH 335/522] Implements the legacy behaviour of the dump When asked if a dump exists we check if it's the current dump, and if it's not then we check on the filesystem for any file matching our `uid.dump` --- .../src/index_controller/dump_actor/actor.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index b41ddadcf..82a38cf96 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -182,12 +182,20 @@ where async fn handle_dump_info(&self, uid: String) -> DumpResult { match &*self.dump_info.lock().await { - None => Err(DumpError::DumpDoesNotExist(uid)), - Some(DumpInfo { uid: ref s, .. }) if &uid != s => Err(DumpError::DumpDoesNotExist(uid)), + None => self.dump_from_fs(uid).await, + Some(DumpInfo { uid: ref s, .. }) if &uid != s => self.dump_from_fs(uid).await, Some(info) => Ok(info.clone()), } } + async fn dump_from_fs(&self, uid: String) -> DumpResult { + self.dump_path + .join(format!("{}.dump", &uid)) + .exists() + .then(|| DumpInfo::new(uid.clone(), DumpStatus::Done)) + .ok_or(DumpError::DumpDoesNotExist(uid)) + } + async fn is_running(&self) -> bool { matches!( *self.dump_info.lock().await, From 529f7962f46873fcdce04125e15da5c485e1a929 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 24 May 2021 15:42:12 +0200 Subject: [PATCH 336/522] handle parallel requests for the dump actor --- .../src/index_controller/dump_actor/actor.rs | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 82a38cf96..fac67cbc0 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -1,7 +1,9 @@ use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus}; use crate::helpers::compression; use crate::index_controller::{index_actor, update_actor, uuid_resolver, IndexMetadata}; +use async_stream::stream; use chrono::Utc; +use futures::stream::StreamExt; use log::{error, info, warn}; use std::{ collections::HashSet, @@ -11,8 +13,10 @@ use std::{ use tokio::sync::{mpsc, Mutex}; use uuid::Uuid; +pub const CONCURRENT_DUMP_MSG: usize = 10; + pub struct DumpActor { - inbox: mpsc::Receiver, + inbox: Option>, inner: InnerDump, } @@ -44,7 +48,7 @@ where dump_path: impl AsRef, ) -> Self { Self { - inbox, + inbox: Some(inbox), inner: InnerDump { uuid_resolver, index, @@ -56,24 +60,41 @@ where } pub async fn run(mut self) { - use DumpMsg::*; - info!("Started dump actor."); - loop { - match self.inbox.recv().await { - Some(CreateDump { ret }) => { - let _ = ret.send(self.inner.clone().handle_create_dump().await); + let mut inbox = self + .inbox + .take() + .expect("Dump Actor must have a inbox at this point."); + + let stream = stream! { + loop { + match inbox.recv().await { + Some(msg) => yield msg, + None => break, } - Some(DumpInfo { ret, uid }) => { - let _ = ret.send(self.inner.handle_dump_info(uid).await); - } - None => break, } - } + }; + + stream + .for_each_concurrent(Some(CONCURRENT_DUMP_MSG), |msg| self.handle_message(msg)) + .await; error!("Dump actor stopped."); } + + async fn handle_message(&self, msg: DumpMsg) { + use DumpMsg::*; + + match msg { + CreateDump { ret } => { + let _ = ret.send(self.inner.clone().handle_create_dump().await); + } + DumpInfo { ret, uid } => { + let _ = ret.send(self.inner.handle_dump_info(uid).await); + } + } + } } impl InnerDump From dcf29e10816ef4b8e48677062206fcfaa889b878 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 24 May 2021 17:33:42 +0200 Subject: [PATCH 337/522] fix the error handling in case there is a panic while creating a dump --- .../src/index_controller/dump_actor/actor.rs | 59 +++++++++++-------- .../src/index_controller/dump_actor/mod.rs | 5 +- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index fac67cbc0..c10cd90b8 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -10,7 +10,7 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{mpsc, oneshot, Mutex}; use uuid::Uuid; pub const CONCURRENT_DUMP_MSG: usize = 10; @@ -88,7 +88,7 @@ where match msg { CreateDump { ret } => { - let _ = ret.send(self.inner.clone().handle_create_dump().await); + let _ = self.inner.clone().handle_create_dump(ret).await; } DumpInfo { ret, uid } => { let _ = ret.send(self.inner.handle_dump_info(uid).await); @@ -103,38 +103,45 @@ where Index: index_actor::IndexActorHandle + Send + Sync + Clone + 'static, Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, { - async fn handle_create_dump(self) -> DumpResult { + async fn handle_create_dump(self, ret: oneshot::Sender>) { if self.is_running().await { - return Err(DumpError::DumpAlreadyRunning); + ret.send(Err(DumpError::DumpAlreadyRunning)) + .expect("Dump actor is dead"); + return; } let uid = generate_uid(); let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress); *self.dump_info.lock().await = Some(info.clone()); - let this = self.clone(); + ret.send(Ok(info)).expect("Dump actor is dead"); - tokio::task::spawn(async move { - match this.perform_dump(uid).await { - Ok(()) => { - if let Some(ref mut info) = *self.dump_info.lock().await { - info.done(); - } else { - warn!("dump actor was in an inconsistant state"); - } - info!("Dump succeed"); - } - Err(e) => { - if let Some(ref mut info) = *self.dump_info.lock().await { - info.with_error(e.to_string()); - } else { - warn!("dump actor was in an inconsistant state"); - } - error!("Dump failed: {}", e); - } - }; - }); + let dump_info = self.dump_info.clone(); + let cloned_uid = uid.clone(); - Ok(info) + let task_result = tokio::task::spawn(self.clone().perform_dump(cloned_uid)).await; + + match task_result { + Ok(Ok(())) => { + if let Some(ref mut info) = *dump_info.lock().await { + info.done(); + } else { + warn!("dump actor was in an inconsistant state"); + } + info!("Dump succeed"); + } + Ok(Err(e)) => { + if let Some(ref mut info) = *dump_info.lock().await { + info.with_error(e.to_string()); + } else { + warn!("dump actor was in an inconsistant state"); + } + error!("Dump failed: {}", e); + } + Err(_) => { + error!("Dump panicked. Dump status set to failed"); + *dump_info.lock().await = Some(DumpInfo::new(uid, DumpStatus::Failed)); + } + }; } async fn perform_dump(self, uid: String) -> anyhow::Result<()> { diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 7d2e5a951..ea0d7adbd 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -13,7 +13,6 @@ use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; #[cfg(test)] use mockall::automock; use serde::{Deserialize, Serialize}; -use serde_json::json; use tempfile::TempDir; use thiserror::Error; use uuid::Uuid; @@ -129,7 +128,7 @@ pub struct DumpInfo { pub uid: String, pub status: DumpStatus, #[serde(skip_serializing_if = "Option::is_none", flatten)] - pub error: Option, + pub error: Option, } impl DumpInfo { @@ -143,7 +142,7 @@ impl DumpInfo { pub fn with_error(&mut self, error: String) { self.status = DumpStatus::Failed; - self.error = Some(json!(error)); + self.error = Some(error); } pub fn done(&mut self) { From 912f0286b332d7b3e1a1840386fcbddc989665a2 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 24 May 2021 18:06:20 +0200 Subject: [PATCH 338/522] remove the dump_inner trickery --- .../src/index_controller/dump_actor/actor.rs | 196 +++++++++--------- 1 file changed, 103 insertions(+), 93 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index c10cd90b8..39d095e9f 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -17,16 +17,11 @@ pub const CONCURRENT_DUMP_MSG: usize = 10; pub struct DumpActor { inbox: Option>, - inner: InnerDump, -} - -#[derive(Clone)] -struct InnerDump { - pub uuid_resolver: UuidResolver, - pub index: Index, - pub update: Update, - pub dump_path: PathBuf, - pub dump_info: Arc>>, + uuid_resolver: UuidResolver, + index: Index, + update: Update, + dump_path: PathBuf, + dump_info: Arc>>, } /// Generate uid from creation date @@ -49,13 +44,11 @@ where ) -> Self { Self { inbox: Some(inbox), - inner: InnerDump { - uuid_resolver, - index, - update, - dump_path: dump_path.as_ref().into(), - dump_info: Arc::new(Mutex::new(None)), - }, + uuid_resolver, + index, + update, + dump_path: dump_path.as_ref().into(), + dump_info: Arc::new(Mutex::new(None)), } } @@ -88,22 +81,15 @@ where match msg { CreateDump { ret } => { - let _ = self.inner.clone().handle_create_dump(ret).await; + let _ = self.handle_create_dump(ret).await; } DumpInfo { ret, uid } => { - let _ = ret.send(self.inner.handle_dump_info(uid).await); + let _ = ret.send(self.handle_dump_info(uid).await); } } } -} -impl InnerDump -where - UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, - Index: index_actor::IndexActorHandle + Send + Sync + Clone + 'static, - Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, -{ - async fn handle_create_dump(self, ret: oneshot::Sender>) { + async fn handle_create_dump(&self, ret: oneshot::Sender>) { if self.is_running().await { ret.send(Err(DumpError::DumpAlreadyRunning)) .expect("Dump actor is dead"); @@ -116,9 +102,15 @@ where ret.send(Ok(info)).expect("Dump actor is dead"); let dump_info = self.dump_info.clone(); - let cloned_uid = uid.clone(); - let task_result = tokio::task::spawn(self.clone().perform_dump(cloned_uid)).await; + let task_result = tokio::task::spawn(perform_dump( + self.dump_path.clone(), + self.uuid_resolver.clone(), + self.index.clone(), + self.update.clone(), + uid.clone(), + )) + .await; match task_result { Ok(Ok(())) => { @@ -144,70 +136,6 @@ where }; } - async fn perform_dump(self, uid: String) -> anyhow::Result<()> { - info!("Performing dump."); - - let dump_dir = self.dump_path.clone(); - tokio::fs::create_dir_all(&dump_dir).await?; - let temp_dump_dir = - tokio::task::spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); - - let uuids = self.uuid_resolver.list().await?; - // maybe we could just keep the vec as-is - let uuids: HashSet<(String, Uuid)> = uuids.into_iter().collect(); - - if uuids.is_empty() { - return Ok(()); - } - - let indexes = self.list_indexes().await?; - - // we create one directory by index - for meta in indexes.iter() { - tokio::fs::create_dir(temp_dump_path.join(&meta.uid)).await?; - } - - let metadata = super::Metadata::new(indexes, env!("CARGO_PKG_VERSION").to_string()); - metadata.to_path(&temp_dump_path).await?; - - self.update.dump(uuids, temp_dump_path.clone()).await?; - - let dump_dir = self.dump_path.clone(); - let dump_path = self.dump_path.join(format!("{}.dump", uid)); - let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; - let temp_dump_file_path = temp_dump_file.path().to_owned(); - compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; - temp_dump_file.persist(&dump_path)?; - Ok(dump_path) - }) - .await??; - - info!("Created dump in {:?}.", dump_path); - - Ok(()) - } - - async fn list_indexes(&self) -> anyhow::Result> { - let uuids = self.uuid_resolver.list().await?; - - let mut ret = Vec::new(); - - for (uid, uuid) in uuids { - let meta = self.index.get_index_meta(uuid).await?; - let meta = IndexMetadata { - uuid, - name: uid.clone(), - uid, - meta, - }; - ret.push(meta); - } - - Ok(ret) - } - async fn handle_dump_info(&self, uid: String) -> DumpResult { match &*self.dump_info.lock().await { None => self.dump_from_fs(uid).await, @@ -234,3 +162,85 @@ where ) } } + +async fn perform_dump( + dump_path: PathBuf, + uuid_resolver: UuidResolver, + index: Index, + update: Update, + uid: String, +) -> anyhow::Result<()> +where + UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, + Index: index_actor::IndexActorHandle + Send + Sync + Clone + 'static, + Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, +{ + info!("Performing dump."); + + let dump_dir = dump_path.clone(); + tokio::fs::create_dir_all(&dump_dir).await?; + let temp_dump_dir = + tokio::task::spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let uuids = uuid_resolver.list().await?; + // maybe we could just keep the vec as-is + let uuids: HashSet<(String, Uuid)> = uuids.into_iter().collect(); + + if uuids.is_empty() { + return Ok(()); + } + + let indexes = list_indexes(&uuid_resolver, &index).await?; + + // we create one directory by index + for meta in indexes.iter() { + tokio::fs::create_dir(temp_dump_path.join(&meta.uid)).await?; + } + + let metadata = super::Metadata::new(indexes, env!("CARGO_PKG_VERSION").to_string()); + metadata.to_path(&temp_dump_path).await?; + + update.dump(uuids, temp_dump_path.clone()).await?; + + let dump_dir = dump_path.clone(); + let dump_path = dump_path.join(format!("{}.dump", uid)); + let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; + let temp_dump_file_path = temp_dump_file.path().to_owned(); + compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; + temp_dump_file.persist(&dump_path)?; + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(()) +} + +async fn list_indexes( + uuid_resolver: &UuidResolver, + index: &Index, +) -> anyhow::Result> +where + UuidResolver: uuid_resolver::UuidResolverHandle, + Index: index_actor::IndexActorHandle, +{ + let uuids = uuid_resolver.list().await?; + + let mut ret = Vec::new(); + + for (uid, uuid) in uuids { + let meta = index.get_index_meta(uuid).await?; + let meta = IndexMetadata { + uuid, + name: uid.clone(), + uid, + meta, + }; + ret.push(meta); + } + + Ok(ret) +} From 49a0e8aa19b5bdf27e3093f3075f461b6047b173 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 24 May 2021 18:19:34 +0200 Subject: [PATCH 339/522] use a RwLock instead of a Mutex --- .../src/index_controller/dump_actor/actor.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 39d095e9f..248526723 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -10,7 +10,7 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use tokio::sync::{mpsc, oneshot, Mutex}; +use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; pub const CONCURRENT_DUMP_MSG: usize = 10; @@ -21,7 +21,7 @@ pub struct DumpActor { index: Index, update: Update, dump_path: PathBuf, - dump_info: Arc>>, + dump_info: Arc>>, } /// Generate uid from creation date @@ -48,7 +48,7 @@ where index, update, dump_path: dump_path.as_ref().into(), - dump_info: Arc::new(Mutex::new(None)), + dump_info: Arc::new(RwLock::new(None)), } } @@ -97,7 +97,7 @@ where } let uid = generate_uid(); let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress); - *self.dump_info.lock().await = Some(info.clone()); + *self.dump_info.write().await = Some(info.clone()); ret.send(Ok(info)).expect("Dump actor is dead"); @@ -114,7 +114,7 @@ where match task_result { Ok(Ok(())) => { - if let Some(ref mut info) = *dump_info.lock().await { + if let Some(ref mut info) = *dump_info.write().await { info.done(); } else { warn!("dump actor was in an inconsistant state"); @@ -122,7 +122,7 @@ where info!("Dump succeed"); } Ok(Err(e)) => { - if let Some(ref mut info) = *dump_info.lock().await { + if let Some(ref mut info) = *dump_info.write().await { info.with_error(e.to_string()); } else { warn!("dump actor was in an inconsistant state"); @@ -131,13 +131,13 @@ where } Err(_) => { error!("Dump panicked. Dump status set to failed"); - *dump_info.lock().await = Some(DumpInfo::new(uid, DumpStatus::Failed)); + *dump_info.write().await = Some(DumpInfo::new(uid, DumpStatus::Failed)); } }; } async fn handle_dump_info(&self, uid: String) -> DumpResult { - match &*self.dump_info.lock().await { + match &*self.dump_info.read().await { None => self.dump_from_fs(uid).await, Some(DumpInfo { uid: ref s, .. }) if &uid != s => self.dump_from_fs(uid).await, Some(info) => Ok(info.clone()), @@ -154,7 +154,7 @@ where async fn is_running(&self) -> bool { matches!( - *self.dump_info.lock().await, + *self.dump_info.read().await, Some(DumpInfo { status: DumpStatus::InProgress, .. From 991d8e1ec618b8e4aa7861041dfad182be4c7b52 Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 25 May 2021 10:48:57 +0200 Subject: [PATCH 340/522] fix the error printing --- meilisearch-http/src/index_controller/dump_actor/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index ea0d7adbd..1508f8eb7 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -127,7 +127,7 @@ pub enum DumpStatus { pub struct DumpInfo { pub uid: String, pub status: DumpStatus, - #[serde(skip_serializing_if = "Option::is_none", flatten)] + #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, } From fe260f13309897655de5c0cea3506016a4658b20 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Tue, 25 May 2021 15:13:47 +0200 Subject: [PATCH 341/522] Update meilisearch-http/src/index_controller/dump_actor/actor.rs Co-authored-by: marin --- meilisearch-http/src/index_controller/dump_actor/actor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 248526723..981b4236d 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -117,7 +117,7 @@ where if let Some(ref mut info) = *dump_info.write().await { info.done(); } else { - warn!("dump actor was in an inconsistant state"); + warn!("Dump actor is in an inconsistent state"); } info!("Dump succeed"); } From 1a6dcec83a517475869700c2b42fee280adaf2fc Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 25 May 2021 15:23:13 +0200 Subject: [PATCH 342/522] crash when the actor have no inbox --- .../src/index_controller/dump_actor/actor.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 248526723..12afb4558 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -4,7 +4,7 @@ use crate::index_controller::{index_actor, update_actor, uuid_resolver, IndexMet use async_stream::stream; use chrono::Utc; use futures::stream::StreamExt; -use log::{error, info, warn}; +use log::{error, info}; use std::{ collections::HashSet, path::{Path, PathBuf}, @@ -114,19 +114,11 @@ where match task_result { Ok(Ok(())) => { - if let Some(ref mut info) = *dump_info.write().await { - info.done(); - } else { - warn!("dump actor was in an inconsistant state"); - } + (*dump_info.write().await).as_mut().expect("Dump actor should have an inbox").done(); info!("Dump succeed"); } Ok(Err(e)) => { - if let Some(ref mut info) = *dump_info.write().await { - info.with_error(e.to_string()); - } else { - warn!("dump actor was in an inconsistant state"); - } + (*dump_info.write().await).as_mut().expect("Dump actor should have an inbox").with_error(e.to_string()); error!("Dump failed: {}", e); } Err(_) => { From 89846d1656183d1f2523dd228f6f6a921e9e084d Mon Sep 17 00:00:00 2001 From: tamo Date: Tue, 25 May 2021 15:47:57 +0200 Subject: [PATCH 343/522] improve panic message --- meilisearch-http/src/index_controller/dump_actor/actor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 12afb4558..8e1e48ebe 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -114,11 +114,11 @@ where match task_result { Ok(Ok(())) => { - (*dump_info.write().await).as_mut().expect("Dump actor should have an inbox").done(); + (*dump_info.write().await).as_mut().expect("Inconsistent dump service state").done(); info!("Dump succeed"); } Ok(Err(e)) => { - (*dump_info.write().await).as_mut().expect("Dump actor should have an inbox").with_error(e.to_string()); + (*dump_info.write().await).as_mut().expect("Inconsistent dump service state").with_error(e.to_string()); error!("Dump failed: {}", e); } Err(_) => { From 2185fb8367f66b69249308814aa65b551198a5a5 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 24 May 2021 16:05:43 +0200 Subject: [PATCH 344/522] dump uuid resolver --- meilisearch-http/src/index_controller/mod.rs | 6 ++-- .../index_controller/uuid_resolver/actor.rs | 17 ++++++--- .../uuid_resolver/handle_impl.rs | 8 +++++ .../index_controller/uuid_resolver/message.rs | 4 +++ .../src/index_controller/uuid_resolver/mod.rs | 35 ++++++++++++++----- .../index_controller/uuid_resolver/store.rs | 33 ++++++++++++++--- 6 files changed, 82 insertions(+), 21 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index d1bb5e170..900482257 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -20,7 +20,7 @@ use dump_actor::DumpActorHandle; use index_actor::IndexActorHandle; use snapshot::{SnapshotService, load_snapshot}; use update_actor::UpdateActorHandle; -use uuid_resolver::{UuidError, UuidResolverHandle}; +use uuid_resolver::{UuidResolverError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; @@ -176,7 +176,7 @@ impl IndexController { match self.uuid_resolver.get(uid).await { Ok(uuid) => Ok(perform_update(uuid).await?), - Err(UuidError::UnexistingIndex(name)) => { + Err(UuidResolverError::UnexistingIndex(name)) => { let uuid = Uuid::new_v4(); let status = perform_update(uuid).await?; // ignore if index creation fails now, since it may already have been created @@ -230,7 +230,7 @@ impl IndexController { match self.uuid_resolver.get(uid).await { Ok(uuid) => Ok(perform_udpate(uuid).await?), - Err(UuidError::UnexistingIndex(name)) if create => { + Err(UuidResolverError::UnexistingIndex(name)) if create => { let uuid = Uuid::new_v4(); let status = perform_udpate(uuid).await?; // ignore if index creation fails now, since it may already have been created diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 253326276..3592c3551 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -4,7 +4,7 @@ use log::{info, warn}; use tokio::sync::mpsc; use uuid::Uuid; -use super::{Result, UuidError, UuidResolveMsg, UuidStore}; +use super::{Result, UuidResolverError, UuidResolveMsg, UuidStore}; pub struct UuidResolverActor { inbox: mpsc::Receiver, @@ -44,6 +44,9 @@ impl UuidResolverActor { Some(GetSize { ret }) => { let _ = ret.send(self.handle_get_size().await); } + Some(DumpRequest { path, ret }) => { + let _ = ret.send(self.handle_dump(path).await); + } // all senders have been dropped, need to quit. None => break, } @@ -54,7 +57,7 @@ impl UuidResolverActor { async fn handle_create(&self, uid: String) -> Result { if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)); + return Err(UuidResolverError::BadlyFormatted(uid)); } self.store.create_uuid(uid, true).await } @@ -63,14 +66,14 @@ impl UuidResolverActor { self.store .get_uuid(uid.clone()) .await? - .ok_or(UuidError::UnexistingIndex(uid)) + .ok_or(UuidResolverError::UnexistingIndex(uid)) } async fn handle_delete(&self, uid: String) -> Result { self.store .delete(uid.clone()) .await? - .ok_or(UuidError::UnexistingIndex(uid)) + .ok_or(UuidResolverError::UnexistingIndex(uid)) } async fn handle_list(&self) -> Result> { @@ -82,9 +85,13 @@ impl UuidResolverActor { self.store.snapshot(path).await } + async fn handle_dump(&self, path: PathBuf) -> Result> { + self.store.dump(path).await + } + async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> { if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)); + return Err(UuidResolverError::BadlyFormatted(uid)); } self.store.insert(uid, uuid).await?; Ok(()) diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index db4c482bd..981beb0f6 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -85,4 +85,12 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .await .expect("Uuid resolver actor has been killed")?) } + async fn dump(&self, path: PathBuf) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::DumpRequest { ret, path }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index a72bf0587..166347455 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -34,4 +34,8 @@ pub enum UuidResolveMsg { GetSize { ret: oneshot::Sender>, }, + DumpRequest { + path: PathBuf, + ret: oneshot::Sender>>, + } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 0cbb2895b..b84025094 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -16,12 +16,12 @@ use store::UuidStore; #[cfg(test)] use mockall::automock; -pub use store::HeedUuidStore; pub use handle_impl::UuidResolverHandleImpl; +pub use store::HeedUuidStore; const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB -pub type Result = std::result::Result; +pub type Result = std::result::Result; #[async_trait::async_trait] #[cfg_attr(test, automock)] @@ -33,20 +33,37 @@ pub trait UuidResolverHandle { async fn list(&self) -> anyhow::Result>; async fn snapshot(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; + async fn dump(&self, path: PathBuf) -> Result>; } #[derive(Debug, Error)] -pub enum UuidError { +pub enum UuidResolverError { #[error("Name already exist.")] NameAlreadyExist, #[error("Index \"{0}\" doesn't exist.")] UnexistingIndex(String), - #[error("Error performing task: {0}")] - TokioTask(#[from] tokio::task::JoinError), - #[error("Database error: {0}")] - Heed(#[from] heed::Error), - #[error("Uuid error: {0}")] - Uuid(#[from] uuid::Error), #[error("Badly formatted index uid: {0}")] BadlyFormatted(String), + #[error("Internal error resolving index uid: {0}")] + Internal(String), } + +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for UuidResolverError { + fn from(other: $other) -> Self { + Self::Internal(other.to_string()) + } + } + )* + } +} + +internal_error!( + heed::Error, + uuid::Error, + std::io::Error, + tokio::task::JoinError, + serde_json::Error +); diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index a781edcba..4fbaa37b4 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -1,5 +1,5 @@ -use std::collections::HashSet; -use std::fs::create_dir_all; +use std::{collections::HashSet, io::Write}; +use std::fs::{create_dir_all, File}; use std::path::{Path, PathBuf}; use heed::{ @@ -8,7 +8,7 @@ use heed::{ }; use uuid::Uuid; -use super::{Result, UuidError, UUID_STORE_SIZE}; +use super::{Result, UuidResolverError, UUID_STORE_SIZE}; use crate::helpers::EnvSizer; #[async_trait::async_trait] @@ -22,6 +22,7 @@ pub trait UuidStore { async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; async fn snapshot(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; + async fn dump(&self, path: PathBuf) -> Result>; } #[derive(Clone)] @@ -48,7 +49,7 @@ impl HeedUuidStore { match db.get(&txn, &name)? { Some(uuid) => { if err { - Err(UuidError::NameAlreadyExist) + Err(UuidResolverError::NameAlreadyExist) } else { let uuid = Uuid::from_slice(uuid)?; Ok(uuid) @@ -138,6 +139,25 @@ impl HeedUuidStore { pub fn get_size(&self) -> Result { Ok(self.env.size()) } + + pub fn dump(&self, path: PathBuf) -> Result> { + let dump_path = path.join("index_uuids"); + create_dir_all(&dump_path)?; + let dump_file_path = dump_path.join("data.jsonl"); + let mut dump_file = File::create(&dump_file_path)?; + let mut uuids = HashSet::new(); + + let txn = self.env.read_txn()?; + for entry in self.db.iter(&txn)? { + let entry = entry?; + let uuid = Uuid::from_slice(entry.1)?; + uuids.insert(uuid); + serde_json::to_writer(&mut dump_file, &serde_json::json!({ "uid": entry.0, "uuid": uuid }))?; + dump_file.write(b"\n").unwrap(); + } + + Ok(uuids) + } } #[async_trait::async_trait] @@ -175,4 +195,9 @@ impl UuidStore for HeedUuidStore { async fn get_size(&self) -> Result { self.get_size() } + + async fn dump(&self, path: PathBuf) -> Result> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.dump(path)).await? + } } From 7ad553670fe0bedac02982dce047b76848761713 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 24 May 2021 17:20:44 +0200 Subject: [PATCH 345/522] index error handling --- .../src/index_controller/index_actor/actor.rs | 91 +++++++++---------- .../src/index_controller/index_actor/mod.rs | 20 +++- .../src/index_controller/index_actor/store.rs | 14 +-- 3 files changed, 63 insertions(+), 62 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 1f0091265..0e2e63468 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -30,10 +30,14 @@ pub struct IndexActor { impl IndexActor { pub fn new(receiver: mpsc::Receiver, store: S) -> IndexResult { let options = IndexerOpts::default(); - let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; + let update_handler = UpdateHandler::new(&options)?; let update_handler = Arc::new(update_handler); let receiver = Some(receiver); - Ok(Self { receiver, update_handler, store }) + Ok(Self { + receiver, + update_handler, + store, + }) } /// `run` poll the write_receiver and read_receiver concurrently, but while messages send @@ -122,7 +126,12 @@ impl IndexActor { Snapshot { uuid, path, ret } => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } - Dump { uid, uuid, path, ret } => { + Dump { + uid, + uuid, + path, + ret, + } => { let _ = ret.send(self.handle_dump(&uid, uuid, path).await); } GetStats { uuid, ret } => { @@ -146,9 +155,7 @@ impl IndexActor { primary_key: Option, ) -> IndexResult { let index = self.store.create(uuid, primary_key).await?; - let meta = spawn_blocking(move || IndexMeta::new(&index)) - .await - .map_err(|e| IndexError::Error(e.into()))??; + let meta = spawn_blocking(move || IndexMeta::new(&index)).await??; Ok(meta) } @@ -165,9 +172,9 @@ impl IndexActor { None => self.store.create(uuid, None).await?, }; - spawn_blocking(move || update_handler.handle_update(meta, data, index)) - .await - .map_err(|e| IndexError::Error(e.into())) + let result = + spawn_blocking(move || update_handler.handle_update(meta, data, index)).await?; + Ok(result) } async fn handle_settings(&self, uuid: Uuid) -> IndexResult> { @@ -176,9 +183,8 @@ impl IndexActor { .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || index.settings().map_err(IndexError::Error)) - .await - .map_err(|e| IndexError::Error(e.into()))? + let result = spawn_blocking(move || index.settings()).await??; + Ok(result) } async fn handle_fetch_documents( @@ -193,13 +199,11 @@ impl IndexActor { .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || { - index - .retrieve_documents(offset, limit, attributes_to_retrieve) - .map_err(IndexError::Error) - }) - .await - .map_err(|e| IndexError::Error(e.into()))? + let result = + spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve)) + .await??; + + Ok(result) } async fn handle_fetch_document( @@ -213,13 +217,12 @@ impl IndexActor { .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || { - index - .retrieve_document(doc_id, attributes_to_retrieve) - .map_err(IndexError::Error) - }) - .await - .map_err(|e| IndexError::Error(e.into()))? + + let result = + spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve)) + .await??; + + Ok(result) } async fn handle_delete(&self, uuid: Uuid) -> IndexResult<()> { @@ -242,9 +245,7 @@ impl IndexActor { async fn handle_get_meta(&self, uuid: Uuid) -> IndexResult { match self.store.get(uuid).await? { Some(index) => { - let meta = spawn_blocking(move || IndexMeta::new(&index)) - .await - .map_err(|e| IndexError::Error(e.into()))??; + let meta = spawn_blocking(move || IndexMeta::new(&index)).await??; Ok(meta) } None => Err(IndexError::UnexistingIndex), @@ -262,7 +263,7 @@ impl IndexActor { .await? .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || match index_settings.primary_key { + let result = spawn_blocking(move || match index_settings.primary_key { Some(ref primary_key) => { let mut txn = index.write_txn()?; if index.primary_key(&txn)?.is_some() { @@ -278,23 +279,22 @@ impl IndexActor { Ok(meta) } }) - .await - .map_err(|e| IndexError::Error(e.into()))? + .await??; + + Ok(result) } async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> IndexResult<()> { use tokio::fs::create_dir_all; path.push("indexes"); - create_dir_all(&path) - .await - .map_err(|e| IndexError::Error(e.into()))?; + create_dir_all(&path).await?; if let Some(index) = self.store.get(uuid).await? { let mut index_path = path.join(format!("index-{}", uuid)); - create_dir_all(&index_path) - .await - .map_err(|e| IndexError::Error(e.into()))?; + + create_dir_all(&index_path).await?; + index_path.push("data.mdb"); spawn_blocking(move || -> anyhow::Result<()> { // Get write txn to wait for ongoing write transaction before snapshot. @@ -304,9 +304,7 @@ impl IndexActor { .copy_to_path(index_path, CompactionOption::Enabled)?; Ok(()) }) - .await - .map_err(|e| IndexError::Error(e.into()))? - .map_err(IndexError::Error)?; + .await??; } Ok(()) @@ -318,9 +316,7 @@ impl IndexActor { use std::io::prelude::*; use tokio::fs::create_dir_all; - create_dir_all(&path) - .await - .map_err(|e| IndexError::Error(e.into()))?; + create_dir_all(&path).await?; if let Some(index) = self.store.get(uuid).await? { let documents_path = path.join(uid).join("documents.jsonl"); @@ -354,9 +350,7 @@ impl IndexActor { Ok(()) }) - .await - .map_err(|e| IndexError::Error(e.into()))? - .map_err(IndexError::Error)?; + .await??; } Ok(()) @@ -379,7 +373,6 @@ impl IndexActor { fields_distribution: index.fields_distribution(&rtxn)?, }) }) - .await - .map_err(|e| IndexError::Error(e.into()))? + .await? } } diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 3b92b1078..fd1d59e8f 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -50,18 +50,30 @@ impl IndexMeta { #[derive(Error, Debug)] pub enum IndexError { - #[error("error with index: {0}")] - Error(#[from] anyhow::Error), #[error("index already exists")] IndexAlreadyExists, #[error("Index doesn't exists")] UnexistingIndex, - #[error("Heed error: {0}")] - HeedError(#[from] heed::Error), #[error("Existing primary key")] ExistingPrimaryKey, + #[error("Internal Index Error: {0}")] + Internal(String) } +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for IndexError { + fn from(other: $other) -> Self { + Self::Internal(other.to_string()) + } + } + )* + } +} + +internal_error!(anyhow::Error, heed::Error, tokio::task::JoinError, std::io::Error); + #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait IndexActorHandle { diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 44f076f2f..3dee166a9 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -56,8 +56,7 @@ impl IndexStore for MapIndexStore { } Ok(index) }) - .await - .map_err(|e| IndexError::Error(e.into()))??; + .await??; self.index_store.write().await.insert(uuid, index.clone()); @@ -78,8 +77,7 @@ impl IndexStore for MapIndexStore { let index_size = self.index_size; let index = spawn_blocking(move || open_index(path, index_size)) - .await - .map_err(|e| IndexError::Error(e.into()))??; + .await??; self.index_store.write().await.insert(uuid, index.clone()); Ok(Some(index)) } @@ -88,18 +86,16 @@ impl IndexStore for MapIndexStore { async fn delete(&self, uuid: Uuid) -> IndexResult> { let db_path = self.path.join(format!("index-{}", uuid)); - fs::remove_dir_all(db_path) - .await - .map_err(|e| IndexError::Error(e.into()))?; + fs::remove_dir_all(db_path).await?; let index = self.index_store.write().await.remove(&uuid); Ok(index) } } fn open_index(path: impl AsRef, size: usize) -> IndexResult { - std::fs::create_dir_all(&path).map_err(|e| IndexError::Error(e.into()))?; + std::fs::create_dir_all(&path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options, &path).map_err(IndexError::Error)?; + let index = milli::Index::new(options, &path)?; Ok(Index(Arc::new(index))) } From 4acbe8e473bac6e506ca71228a0d405551934e2f Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 24 May 2021 18:16:35 +0200 Subject: [PATCH 346/522] implement index dump --- meilisearch-http/src/index/mod.rs | 61 ++++++++++++++++++- .../dump_actor/handle_impl.rs | 2 +- .../src/index_controller/index_actor/actor.rs | 58 ++++-------------- .../index_actor/handle_impl.rs | 4 +- .../index_controller/index_actor/message.rs | 1 - .../src/index_controller/index_actor/mod.rs | 2 +- .../update_actor/update_store.rs | 2 +- .../index_controller/uuid_resolver/store.rs | 4 +- 8 files changed, 79 insertions(+), 55 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index f26cc4283..c4bf19856 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,8 +1,11 @@ -use std::{collections::{BTreeSet, HashSet}, marker::PhantomData}; +use std::{collections::{BTreeSet, HashSet}, io::Write, marker::PhantomData, path::{Path, PathBuf}}; use std::ops::Deref; use std::sync::Arc; +use std::fs::File; use anyhow::{bail, Context}; +use heed::RoTxn; +use indexmap::IndexMap; use milli::obkv_to_json; use serde_json::{Map, Value}; @@ -38,7 +41,10 @@ where impl Index { pub fn settings(&self) -> anyhow::Result> { let txn = self.read_txn()?; + self.settings_txn(&txn) + } + pub fn settings_txn(&self, txn: &RoTxn) -> anyhow::Result> { let displayed_attributes = self .displayed_fields(&txn)? .map(|fields| fields.into_iter().map(String::from).collect()); @@ -161,4 +167,57 @@ impl Index { displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid)); Ok(displayed_fields_ids) } + + pub fn dump(&self, path: PathBuf) -> anyhow::Result<()> { + // acquire write txn make sure any ongoing write is finnished before we start. + let txn = self.env.write_txn()?; + + self.dump_documents(&txn, &path)?; + self.dump_meta(&txn, &path)?; + + Ok(()) + } + + fn dump_documents(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { + println!("dumping documents"); + let document_file_path = path.as_ref().join("documents.jsonl"); + let mut document_file = File::create(&document_file_path)?; + + let documents = self.all_documents(txn)?; + let fields_ids_map = self.fields_ids_map(txn)?; + + // dump documents + let mut json_map = IndexMap::new(); + for document in documents { + let (_, reader) = document?; + + for (fid, bytes) in reader.iter() { + if let Some(name) = fields_ids_map.name(fid) { + json_map.insert(name, serde_json::from_slice::(bytes)?); + } + } + + serde_json::to_writer(&mut document_file, &json_map)?; + document_file.write(b"\n")?; + + json_map.clear(); + } + + Ok(()) + } + + fn dump_meta(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { + println!("dumping settings"); + let meta_file_path = path.as_ref().join("meta.json"); + let mut meta_file = File::create(&meta_file_path)?; + + let settings = self.settings_txn(txn)?; + let json = serde_json::json!({ + "settings": settings, + }); + + serde_json::to_writer(&mut meta_file, &json)?; + + Ok(()) + } } diff --git a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs index 601c97c01..575119410 100644 --- a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs @@ -1,4 +1,4 @@ -use std::path::{Path}; +use std::path::Path; use actix_web::web::Bytes; use tokio::sync::{mpsc, oneshot}; use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg, DumpResult}; diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 0e2e63468..f6f7cdc28 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -6,7 +6,7 @@ use async_stream::stream; use futures::stream::StreamExt; use heed::CompactionOption; use log::debug; -use tokio::sync::mpsc; +use tokio::{fs, sync::mpsc}; use tokio::task::spawn_blocking; use uuid::Uuid; @@ -126,13 +126,8 @@ impl IndexActor { Snapshot { uuid, path, ret } => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } - Dump { - uid, - uuid, - path, - ret, - } => { - let _ = ret.send(self.handle_dump(&uid, uuid, path).await); + Dump { uuid, path, ret } => { + let _ = ret.send(self.handle_dump(uuid, path).await); } GetStats { uuid, ret } => { let _ = ret.send(self.handle_get_stats(uuid).await); @@ -312,46 +307,17 @@ impl IndexActor { /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the /// documents and all the settings. - async fn handle_dump(&self, uid: &str, uuid: Uuid, path: PathBuf) -> IndexResult<()> { - use std::io::prelude::*; - use tokio::fs::create_dir_all; + async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; - create_dir_all(&path).await?; + let path = path.join(format!("indexes/index-{}/", uuid)); + fs::create_dir_all(&path).await?; - if let Some(index) = self.store.get(uuid).await? { - let documents_path = path.join(uid).join("documents.jsonl"); - let settings_path = path.join(uid).join("settings.json"); - - spawn_blocking(move || -> anyhow::Result<()> { - // first we dump all the documents - let file = File::create(documents_path)?; - let mut file = std::io::BufWriter::new(file); - - // Get write txn to wait for ongoing write transaction before dump. - let txn = index.write_txn()?; - let fields_ids_map = index.fields_ids_map(&txn)?; - // we want to save **all** the fields in the dump. - let fields_to_dump: Vec = fields_ids_map.iter().map(|(id, _)| id).collect(); - - for document in index.all_documents(&txn)? { - let (_doc_id, document) = document?; - let json = milli::obkv_to_json(&fields_to_dump, &fields_ids_map, document)?; - file.write_all(serde_json::to_string(&json)?.as_bytes())?; - file.write_all(b"\n")?; - } - - // then we dump all the settings - let file = File::create(settings_path)?; - let mut file = std::io::BufWriter::new(file); - let settings = index.settings()?; - - file.write_all(serde_json::to_string(&settings)?.as_bytes())?; - file.write_all(b"\n")?; - - Ok(()) - }) - .await??; - } + tokio::task::spawn_blocking(move || index.dump(path)).await??; Ok(()) } diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 64b63e5f0..26aa189d0 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -136,9 +136,9 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Dump { uid, uuid, path, ret }; + let msg = IndexMsg::Dump { uuid, path, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 37faa1e31..714a30ecc 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -61,7 +61,6 @@ pub enum IndexMsg { ret: oneshot::Sender>, }, Dump { - uid: String, uuid: Uuid, path: PathBuf, ret: oneshot::Sender>, diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index fd1d59e8f..dbea5151d 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -109,7 +109,7 @@ pub trait IndexActorHandle { index_settings: IndexSettings, ) -> IndexResult; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; - async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()>; + async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; async fn get_index_stats(&self, uuid: Uuid) -> IndexResult; } diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index f91a2740c..d22be0bd4 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -642,7 +642,7 @@ impl UpdateStore { let path = &path; let mut stream = futures::stream::iter(uuids.iter()) - .map(|(uid, uuid)| handle.dump(uid.clone(), *uuid, path.clone())) + .map(|(uid, uuid)| handle.dump(*uuid, path.clone())) .buffer_unordered(CONCURRENT_INDEX_MSG / 3); Handle::current().block_on(async { diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 4fbaa37b4..b497116cb 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -152,8 +152,8 @@ impl HeedUuidStore { let entry = entry?; let uuid = Uuid::from_slice(entry.1)?; uuids.insert(uuid); - serde_json::to_writer(&mut dump_file, &serde_json::json!({ "uid": entry.0, "uuid": uuid }))?; - dump_file.write(b"\n").unwrap(); + serde_json::to_writer(&mut dump_file, &serde_json::json!({ "uid": entry.0, "uuid": uuid + }))?; dump_file.write(b"\n").unwrap(); } Ok(uuids) From 464639aa0f331e1720711358b1e8e9e055822eaa Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 25 May 2021 09:46:11 +0200 Subject: [PATCH 347/522] udpate actor error improvements --- meilisearch-http/src/index_controller/mod.rs | 8 +- .../index_controller/update_actor/actor.rs | 87 +++++++------------ .../src/index_controller/update_actor/mod.rs | 30 ++++++- 3 files changed, 58 insertions(+), 67 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 900482257..61bc71114 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -158,13 +158,7 @@ impl IndexController { // prevent dead_locking between the update_handle::update that waits for the update to be // registered and the update_actor that waits for the the payload to be sent to it. tokio::task::spawn_local(async move { - payload - .map(|bytes| { - bytes.map_err(|e| { - Box::new(e) as Box - }) - }) - .for_each(|r| async { + payload.for_each(|r| async { let _ = sender.send(r).await; }) .await diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index f576ce7a8..27906a1a8 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -11,7 +11,7 @@ use tokio::sync::mpsc; use uuid::Uuid; use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore, UpdateStoreInfo}; -use crate::index_controller::index_actor::{IndexActorHandle}; +use crate::index_controller::index_actor::IndexActorHandle; use crate::index_controller::{UpdateMeta, UpdateStatus}; pub struct UpdateActor { @@ -42,7 +42,12 @@ where let store = UpdateStore::open(options, &path, index_handle.clone())?; std::fs::create_dir_all(path.join("update_files"))?; assert!(path.exists()); - Ok(Self { path, store, inbox, index_handle }) + Ok(Self { + path, + store, + inbox, + index_handle, + }) } pub async fn run(mut self) { @@ -90,9 +95,7 @@ where mut payload: mpsc::Receiver>, ) -> Result { let file_path = match meta { - UpdateMeta::DocumentsAddition { .. } - | UpdateMeta::DeleteDocuments => { - + UpdateMeta::DocumentsAddition { .. } | UpdateMeta::DeleteDocuments => { let update_file_id = uuid::Uuid::new_v4(); let path = self .path @@ -102,39 +105,26 @@ where .write(true) .create(true) .open(&path) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; + .await?; let mut file_len = 0; while let Some(bytes) = payload.recv().await { - match bytes { - Ok(bytes) => { - file_len += bytes.as_ref().len(); - file.write_all(bytes.as_ref()) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; - } - Err(e) => { - return Err(UpdateError::Error(e)); - } - } + let bytes = bytes?; + file_len += bytes.as_ref().len(); + file.write_all(bytes.as_ref()).await?; } if file_len != 0 { - file.flush() - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; + file.flush().await?; let file = file.into_std().await; Some((file, path)) } else { // empty update, delete the empty file. - fs::remove_file(&path) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))?; + fs::remove_file(&path).await?; None } } - _ => None + _ => None, }; let update_store = self.store.clone(); @@ -145,17 +135,15 @@ where // If the payload is empty, ignore the check. let path = if let Some((mut file, path)) = file_path { // set the file back to the beginning - file.seek(SeekFrom::Start(0)).map_err(|e| UpdateError::Error(Box::new(e)))?; + file.seek(SeekFrom::Start(0))?; // Check that the json payload is valid: let reader = BufReader::new(&mut file); let mut checker = JsonChecker::new(reader); if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() { // The json file is invalid, we use Serde to get a nice error message: - file.seek(SeekFrom::Start(0)) - .map_err(|e| UpdateError::Error(Box::new(e)))?; - let _: serde_json::Value = serde_json::from_reader(file) - .map_err(|e| UpdateError::Error(Box::new(e)))?; + file.seek(SeekFrom::Start(0))?; + let _: serde_json::Value = serde_json::from_reader(file)?; } Some(path) } else { @@ -163,32 +151,27 @@ where }; // The payload is valid, we can register it to the update store. - update_store + let status = update_store .register_update(meta, path, uuid) - .map(UpdateStatus::Enqueued) - .map_err(|e| UpdateError::Error(Box::new(e))) + .map(UpdateStatus::Enqueued)?; + Ok(status) }) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))? + .await? } async fn handle_list_updates(&self, uuid: Uuid) -> Result> { let update_store = self.store.clone(); tokio::task::spawn_blocking(move || { - let result = update_store - .list(uuid) - .map_err(|e| UpdateError::Error(e.into()))?; + let result = update_store.list(uuid)?; Ok(result) }) - .await - .map_err(|e| UpdateError::Error(Box::new(e)))? + .await? } async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { let store = self.store.clone(); let result = store - .meta(uuid, id) - .map_err(|e| UpdateError::Error(Box::new(e)))? + .meta(uuid, id)? .ok_or(UpdateError::UnexistingUpdate(id))?; Ok(result) } @@ -196,10 +179,7 @@ where async fn handle_delete(&self, uuid: Uuid) -> Result<()> { let store = self.store.clone(); - tokio::task::spawn_blocking(move || store.delete_all(uuid)) - .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?; + tokio::task::spawn_blocking(move || store.delete_all(uuid)).await??; Ok(()) } @@ -208,10 +188,8 @@ where let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); - tokio::task::spawn_blocking(move || update_store.snapshot(&uuids, &path, index_handle)) - .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?; + tokio::task::spawn_blocking(move || update_store.snapshot(&uuids, &path, index_handle)) + .await??; Ok(()) } @@ -223,9 +201,8 @@ where update_store.dump(&uuids, path.to_path_buf(), index_handle)?; Ok(()) }) - .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?; + .await??; + Ok(()) } @@ -235,9 +212,7 @@ where let info = update_store.get_info()?; Ok(info) }) - .await - .map_err(|e| UpdateError::Error(e.into()))? - .map_err(|e| UpdateError::Error(e.into()))?; + .await??; Ok(info) } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 05b793e45..a0c498e92 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -5,6 +5,7 @@ mod update_store; use std::{collections::HashSet, path::PathBuf}; +use actix_http::error::PayloadError; use thiserror::Error; use tokio::sync::mpsc; use uuid::Uuid; @@ -14,23 +15,44 @@ use crate::index_controller::{UpdateMeta, UpdateStatus}; use actor::UpdateActor; use message::UpdateMsg; -pub use update_store::{UpdateStore, UpdateStoreInfo}; pub use handle_impl::UpdateActorHandleImpl; +pub use update_store::{UpdateStore, UpdateStoreInfo}; pub type Result = std::result::Result; -type PayloadData = std::result::Result>; +type PayloadData = std::result::Result; #[cfg(test)] use mockall::automock; #[derive(Debug, Error)] pub enum UpdateError { - #[error("error with update: {0}")] - Error(Box), #[error("Update {0} doesn't exist.")] UnexistingUpdate(u64), + #[error("Internal error processing update: {0}")] + Internal(String), } +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for UpdateError { + fn from(other: $other) -> Self { + Self::Internal(other.to_string()) + } + } + )* + } +} + +internal_error!( + heed::Error, + std::io::Error, + serde_json::Error, + PayloadError, + tokio::task::JoinError, + anyhow::Error +); + #[async_trait::async_trait] #[cfg_attr(test, automock(type Data=Vec;))] pub trait UpdateActorHandle { From 3593ebb8aab919014433b3a0d62a95f593a25f56 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 25 May 2021 16:33:09 +0200 Subject: [PATCH 348/522] dump updates --- meilisearch-http/src/index/updates.rs | 1 + .../index_controller/update_actor/actor.rs | 3 +- .../update_actor/handle_impl.rs | 2 +- .../index_controller/update_actor/message.rs | 2 +- .../src/index_controller/update_actor/mod.rs | 6 +- .../update_actor/store/codec.rs | 86 +++++++++ .../update_actor/store/dump.rs | 146 +++++++++++++++ .../{update_store.rs => store/mod.rs} | 175 ++---------------- 8 files changed, 252 insertions(+), 169 deletions(-) create mode 100644 meilisearch-http/src/index_controller/update_actor/store/codec.rs create mode 100644 meilisearch-http/src/index_controller/update_actor/store/dump.rs rename meilisearch-http/src/index_controller/update_actor/{update_store.rs => store/mod.rs} (79%) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 0f4bf3589..2b489451b 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -178,6 +178,7 @@ impl Index { let indexing_callback = |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); + let gzipped = false; let addition = match content { Some(content) if gzipped => { diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 27906a1a8..4097f31aa 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -194,9 +194,10 @@ where Ok(()) } - async fn handle_dump(&self, uuids: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()> { + async fn handle_dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); + println!("starting dump"); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { update_store.dump(&uuids, path.to_path_buf(), index_handle)?; Ok(()) diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index a497a3c5c..cc5ba9757 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -71,7 +71,7 @@ where receiver.await.expect("update actor killed.") } - async fn dump(&self, uuids: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()> { + async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Dump { uuids, path, ret }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 4103ca121..37df2af32 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -32,7 +32,7 @@ pub enum UpdateMsg { ret: oneshot::Sender>, }, Dump { - uuids: HashSet<(String, Uuid)>, + uuids: HashSet, path: PathBuf, ret: oneshot::Sender>, }, diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index a0c498e92..8cd77e252 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -1,7 +1,7 @@ mod actor; mod handle_impl; mod message; -mod update_store; +mod store; use std::{collections::HashSet, path::PathBuf}; @@ -16,7 +16,7 @@ use actor::UpdateActor; use message::UpdateMsg; pub use handle_impl::UpdateActorHandleImpl; -pub use update_store::{UpdateStore, UpdateStoreInfo}; +pub use store::{UpdateStore, UpdateStoreInfo}; pub type Result = std::result::Result; type PayloadData = std::result::Result; @@ -62,7 +62,7 @@ pub trait UpdateActorHandle { async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; async fn snapshot(&self, uuid: HashSet, path: PathBuf) -> Result<()>; - async fn dump(&self, uuid: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()>; + async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()>; async fn get_info(&self) -> Result; async fn update( &self, diff --git a/meilisearch-http/src/index_controller/update_actor/store/codec.rs b/meilisearch-http/src/index_controller/update_actor/store/codec.rs new file mode 100644 index 000000000..e07b52eec --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/store/codec.rs @@ -0,0 +1,86 @@ +use std::{borrow::Cow, convert::TryInto, mem::size_of}; + +use heed::{BytesDecode, BytesEncode}; +use uuid::Uuid; + +pub struct NextIdCodec; + +pub enum NextIdKey { + Global, + Index(Uuid), +} + +impl<'a> BytesEncode<'a> for NextIdCodec { + type EItem = NextIdKey; + + fn bytes_encode(item: &'a Self::EItem) -> Option> { + match item { + NextIdKey::Global => Some(Cow::Borrowed(b"__global__")), + NextIdKey::Index(ref uuid) => Some(Cow::Borrowed(uuid.as_bytes())), + } + } +} + +pub struct PendingKeyCodec; + +impl<'a> BytesEncode<'a> for PendingKeyCodec { + type EItem = (u64, Uuid, u64); + + fn bytes_encode((global_id, uuid, update_id): &'a Self::EItem) -> Option> { + let mut bytes = Vec::with_capacity(size_of::()); + bytes.extend_from_slice(&global_id.to_be_bytes()); + bytes.extend_from_slice(uuid.as_bytes()); + bytes.extend_from_slice(&update_id.to_be_bytes()); + Some(Cow::Owned(bytes)) + } +} + +impl<'a> BytesDecode<'a> for PendingKeyCodec { + type DItem = (u64, Uuid, u64); + + fn bytes_decode(bytes: &'a [u8]) -> Option { + let global_id_bytes = bytes.get(0..size_of::())?.try_into().ok()?; + let global_id = u64::from_be_bytes(global_id_bytes); + + let uuid_bytes = bytes + .get(size_of::()..(size_of::() + size_of::()))? + .try_into() + .ok()?; + let uuid = Uuid::from_bytes(uuid_bytes); + + let update_id_bytes = bytes + .get((size_of::() + size_of::())..)? + .try_into() + .ok()?; + let update_id = u64::from_be_bytes(update_id_bytes); + + Some((global_id, uuid, update_id)) + } +} + +pub struct UpdateKeyCodec; + +impl<'a> BytesEncode<'a> for UpdateKeyCodec { + type EItem = (Uuid, u64); + + fn bytes_encode((uuid, update_id): &'a Self::EItem) -> Option> { + let mut bytes = Vec::with_capacity(size_of::()); + bytes.extend_from_slice(uuid.as_bytes()); + bytes.extend_from_slice(&update_id.to_be_bytes()); + Some(Cow::Owned(bytes)) + } +} + +impl<'a> BytesDecode<'a> for UpdateKeyCodec { + type DItem = (Uuid, u64); + + fn bytes_decode(bytes: &'a [u8]) -> Option { + let uuid_bytes = bytes.get(0..size_of::())?.try_into().ok()?; + let uuid = Uuid::from_bytes(uuid_bytes); + + let update_id_bytes = bytes.get(size_of::()..)?.try_into().ok()?; + let update_id = u64::from_be_bytes(update_id_bytes); + + Some((uuid, update_id)) + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs new file mode 100644 index 000000000..8b75f9e5d --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -0,0 +1,146 @@ +use std::{ + collections::HashSet, + fs::{copy, create_dir_all, File}, + io::Write, + path::{Path, PathBuf}, +}; + +use anyhow::Context; +use heed::RoTxn; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{State, codec::UpdateKeyCodec}; +use super::UpdateStore; +use crate::index_controller::{index_actor::IndexActorHandle, UpdateStatus}; + +#[derive(Serialize, Deserialize)] +struct UpdateEntry { + uuid: Uuid, + update: UpdateStatus, +} + +impl UpdateStore { + pub fn dump( + &self, + uuids: &HashSet, + path: PathBuf, + handle: impl IndexActorHandle, + ) -> anyhow::Result<()> { + let state_lock = self.state.write(); + state_lock.swap(State::Dumping); + + // txn must *always* be acquired after state lock, or it will dead lock. + let txn = self.env.write_txn()?; + + let dump_path = path.join("updates"); + create_dir_all(&dump_path)?; + + self.dump_updates(&txn, uuids, &dump_path)?; + + let fut = dump_indexes(uuids, handle, &path); + tokio::runtime::Handle::current().block_on(fut)?; + + state_lock.swap(State::Idle); + + Ok(()) + } + + fn dump_updates( + &self, + txn: &RoTxn, + uuids: &HashSet, + path: impl AsRef, + ) -> anyhow::Result<()> { + let dump_data_path = path.as_ref().join("data.jsonl"); + let mut dump_data_file = File::create(dump_data_path)?; + + let update_files_path = path.as_ref().join("update_files"); + create_dir_all(&update_files_path)?; + + self.dump_pending(&txn, uuids, &mut dump_data_file, &update_files_path)?; + self.dump_completed(&txn, uuids, &mut dump_data_file)?; + + Ok(()) + } + + fn dump_pending( + &self, + txn: &RoTxn, + uuids: &HashSet, + mut file: &mut File, + update_files_path: impl AsRef, + ) -> anyhow::Result<()> { + let pendings = self.pending_queue.iter(txn)?.lazily_decode_data(); + + for pending in pendings { + let ((_, uuid, _), data) = pending?; + if uuids.contains(&uuid) { + let mut update = data.decode()?; + + if let Some(content) = update.content.take() { + update.content = Some(dump_update_file(content, &update_files_path)?); + } + + let update_json = UpdateEntry { + uuid, + update: update.into(), + }; + + serde_json::to_writer(&mut file, &update_json)?; + file.write(b"\n")?; + } + } + + Ok(()) + } + + fn dump_completed( + &self, + txn: &RoTxn, + uuids: &HashSet, + mut file: &mut File, + ) -> anyhow::Result<()> { + let updates = self + .updates + .iter(txn)? + .remap_key_type::() + .lazily_decode_data(); + + for update in updates { + let ((uuid, _), data) = update?; + if uuids.contains(&uuid) { + let update = data.decode()?.into(); + + let update_json = UpdateEntry { uuid, update }; + + serde_json::to_writer(&mut file, &update_json)?; + file.write(b"\n")?; + } + } + + Ok(()) + } +} + +async fn dump_indexes(uuids: &HashSet, handle: impl IndexActorHandle, path: impl AsRef)-> anyhow::Result<()> { + for uuid in uuids { + handle.dump(*uuid, path.as_ref().to_owned()).await?; + } + + Ok(()) +} + +fn dump_update_file( + file_path: impl AsRef, + dump_path: impl AsRef, +) -> anyhow::Result { + let filename: PathBuf = file_path + .as_ref() + .file_name() + .context("invalid update file name")? + .into(); + let dump_file_path = dump_path.as_ref().join(&filename); + copy(file_path, dump_file_path)?; + Ok(filename) +} diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs similarity index 79% rename from meilisearch-http/src/index_controller/update_actor/update_store.rs rename to meilisearch-http/src/index_controller/update_actor/store/mod.rs index d22be0bd4..52bd8d62a 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -1,23 +1,25 @@ +mod dump; +mod codec; + use std::collections::{BTreeMap, HashSet}; -use std::convert::TryInto; use std::fs::{copy, create_dir_all, remove_file, File}; -use std::mem::size_of; use std::path::Path; use std::sync::Arc; -use std::{borrow::Cow, path::PathBuf}; use anyhow::Context; use arc_swap::ArcSwap; use futures::StreamExt; use heed::types::{ByteSlice, OwnedType, SerdeJson}; use heed::zerocopy::U64; -use heed::{BytesDecode, BytesEncode, CompactionOption, Database, Env, EnvOpenOptions}; +use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use log::error; use parking_lot::{Mutex, MutexGuard}; use tokio::runtime::Handle; use tokio::sync::mpsc; use uuid::Uuid; +use codec::*; + use super::UpdateMeta; use crate::{helpers::EnvSizer, index_controller::index_actor::IndexResult}; use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle}; @@ -25,13 +27,6 @@ use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, Ind #[allow(clippy::upper_case_acronyms)] type BEU64 = U64; -struct NextIdCodec; - -enum NextIdKey { - Global, - Index(Uuid), -} - pub struct UpdateStoreInfo { /// Size of the update store in bytes. pub size: u64, @@ -45,13 +40,13 @@ pub struct StateLock { data: ArcSwap, } -struct StateLockGuard<'a> { +pub struct StateLockGuard<'a> { _lock: MutexGuard<'a, ()>, state: &'a StateLock, } impl StateLockGuard<'_> { - fn swap(&self, state: State) -> Arc { + pub fn swap(&self, state: State) -> Arc { self.state.data.swap(Arc::new(state)) } } @@ -63,11 +58,11 @@ impl StateLock { Self { lock, data } } - fn read(&self) -> Arc { + pub fn read(&self) -> Arc { self.data.load().clone() } - fn write(&self) -> StateLockGuard { + pub fn write(&self) -> StateLockGuard { let _lock = self.lock.lock(); let state = &self; StateLockGuard { _lock, state } @@ -82,81 +77,6 @@ pub enum State { Dumping, } -impl<'a> BytesEncode<'a> for NextIdCodec { - type EItem = NextIdKey; - - fn bytes_encode(item: &'a Self::EItem) -> Option> { - match item { - NextIdKey::Global => Some(Cow::Borrowed(b"__global__")), - NextIdKey::Index(ref uuid) => Some(Cow::Borrowed(uuid.as_bytes())), - } - } -} - -struct PendingKeyCodec; - -impl<'a> BytesEncode<'a> for PendingKeyCodec { - type EItem = (u64, Uuid, u64); - - fn bytes_encode((global_id, uuid, update_id): &'a Self::EItem) -> Option> { - let mut bytes = Vec::with_capacity(size_of::()); - bytes.extend_from_slice(&global_id.to_be_bytes()); - bytes.extend_from_slice(uuid.as_bytes()); - bytes.extend_from_slice(&update_id.to_be_bytes()); - Some(Cow::Owned(bytes)) - } -} - -impl<'a> BytesDecode<'a> for PendingKeyCodec { - type DItem = (u64, Uuid, u64); - - fn bytes_decode(bytes: &'a [u8]) -> Option { - let global_id_bytes = bytes.get(0..size_of::())?.try_into().ok()?; - let global_id = u64::from_be_bytes(global_id_bytes); - - let uuid_bytes = bytes - .get(size_of::()..(size_of::() + size_of::()))? - .try_into() - .ok()?; - let uuid = Uuid::from_bytes(uuid_bytes); - - let update_id_bytes = bytes - .get((size_of::() + size_of::())..)? - .try_into() - .ok()?; - let update_id = u64::from_be_bytes(update_id_bytes); - - Some((global_id, uuid, update_id)) - } -} - -struct UpdateKeyCodec; - -impl<'a> BytesEncode<'a> for UpdateKeyCodec { - type EItem = (Uuid, u64); - - fn bytes_encode((uuid, update_id): &'a Self::EItem) -> Option> { - let mut bytes = Vec::with_capacity(size_of::()); - bytes.extend_from_slice(uuid.as_bytes()); - bytes.extend_from_slice(&update_id.to_be_bytes()); - Some(Cow::Owned(bytes)) - } -} - -impl<'a> BytesDecode<'a> for UpdateKeyCodec { - type DItem = (Uuid, u64); - - fn bytes_decode(bytes: &'a [u8]) -> Option { - let uuid_bytes = bytes.get(0..size_of::())?.try_into().ok()?; - let uuid = Uuid::from_bytes(uuid_bytes); - - let update_id_bytes = bytes.get(size_of::()..)?.try_into().ok()?; - let update_id = u64::from_be_bytes(update_id_bytes); - - Some((uuid, update_id)) - } -} - #[derive(Clone)] pub struct UpdateStore { pub env: Env, @@ -174,7 +94,7 @@ pub struct UpdateStore { /// | 16-bytes | 8-bytes | updates: Database>, /// Indicates the current state of the update store, - state: Arc, + pub state: Arc, /// Wake up the loop when a new event occurs. notification_sender: mpsc::Sender<()>, } @@ -364,6 +284,7 @@ impl UpdateStore { let processing = pending.processing(); // Acquire the state lock and set the current state to processing. + // txn must *always* be acquired after state lock, or it will dead lock. let state = self.state.write(); state.swap(State::Processing(index_uuid, processing.clone())); @@ -580,78 +501,6 @@ impl UpdateStore { Ok(()) } - pub fn dump( - &self, - uuids: &HashSet<(String, Uuid)>, - path: PathBuf, - handle: impl IndexActorHandle, - ) -> anyhow::Result<()> { - use std::io::prelude::*; - let state_lock = self.state.write(); - state_lock.swap(State::Dumping); - - let txn = self.env.write_txn()?; - - for (index_uid, index_uuid) in uuids.iter() { - let file = File::create(path.join(index_uid).join("updates.jsonl"))?; - let mut file = std::io::BufWriter::new(file); - - let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); - for entry in pendings { - let ((_, uuid, _), pending) = entry?; - if &uuid == index_uuid { - let mut update: UpdateStatus = pending.decode()?.into(); - if let Some(path) = update.content_path_mut() { - *path = path.file_name().expect("update path can't be empty").into(); - } - serde_json::to_writer(&mut file, &update)?; - file.write_all(b"\n")?; - } - } - - let updates = self.updates.prefix_iter(&txn, index_uuid.as_bytes())?; - for entry in updates { - let (_, update) = entry?; - let mut update = update.clone(); - if let Some(path) = update.content_path_mut() { - *path = path.file_name().expect("update path can't be empty").into(); - } - serde_json::to_writer(&mut file, &update)?; - file.write_all(b"\n")?; - } - } - - let update_files_path = path.join("update_files"); - create_dir_all(&update_files_path)?; - - let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); - - for entry in pendings { - let ((_, uuid, _), pending) = entry?; - if uuids.iter().any(|(_, id)| id == &uuid) { - if let Some(path) = pending.decode()?.content_path() { - let name = path.file_name().unwrap(); - let to = update_files_path.join(name); - copy(path, to)?; - } - } - } - - // Perform the dump of each index concurently. Only a third of the capabilities of - // the index actor at a time not to put too much pressure on the index actor - let path = &path; - - let mut stream = futures::stream::iter(uuids.iter()) - .map(|(uid, uuid)| handle.dump(*uuid, path.clone())) - .buffer_unordered(CONCURRENT_INDEX_MSG / 3); - - Handle::current().block_on(async { - while let Some(res) = stream.next().await { - res?; - } - Ok(()) - }) - } pub fn get_info(&self) -> anyhow::Result { let mut size = self.env.size(); From 9278a6fe5906f0246d6c4c4d4bf07873ab2763c9 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 25 May 2021 18:14:11 +0200 Subject: [PATCH 349/522] integrate in dump actor --- .../src/index_controller/dump_actor/actor.rs | 37 ++++--------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 8e1e48ebe..2d931dcbd 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -106,7 +106,6 @@ where let task_result = tokio::task::spawn(perform_dump( self.dump_path.clone(), self.uuid_resolver.clone(), - self.index.clone(), self.update.clone(), uid.clone(), )) @@ -155,50 +154,28 @@ where } } -async fn perform_dump( +async fn perform_dump( dump_path: PathBuf, uuid_resolver: UuidResolver, - index: Index, - update: Update, + update_handle: Update, uid: String, ) -> anyhow::Result<()> where UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, - Index: index_actor::IndexActorHandle + Send + Sync + Clone + 'static, Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, { info!("Performing dump."); - let dump_dir = dump_path.clone(); - tokio::fs::create_dir_all(&dump_dir).await?; - let temp_dump_dir = - tokio::task::spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); + let dump_path_clone = dump_path.clone(); + let temp_dump_path = tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(dump_path_clone)).await??; - let uuids = uuid_resolver.list().await?; - // maybe we could just keep the vec as-is - let uuids: HashSet<(String, Uuid)> = uuids.into_iter().collect(); + let uuids = uuid_resolver.dump(temp_dump_path.path().to_owned()).await?; - if uuids.is_empty() { - return Ok(()); - } + update_handle.dump(uuids, temp_dump_path.path().to_owned()).await?; - let indexes = list_indexes(&uuid_resolver, &index).await?; - - // we create one directory by index - for meta in indexes.iter() { - tokio::fs::create_dir(temp_dump_path.join(&meta.uid)).await?; - } - - let metadata = super::Metadata::new(indexes, env!("CARGO_PKG_VERSION").to_string()); - metadata.to_path(&temp_dump_path).await?; - - update.dump(uuids, temp_dump_path.clone()).await?; - - let dump_dir = dump_path.clone(); let dump_path = dump_path.join(format!("{}.dump", uid)); let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; + let temp_dump_file = tempfile::NamedTempFile::new_in(&dump_path)?; let temp_dump_file_path = temp_dump_file.path().to_owned(); compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; temp_dump_file.persist(&dump_path)?; From e818c33fec0cb8e8af281335dd40150bca1ad1ce Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 26 May 2021 20:42:09 +0200 Subject: [PATCH 350/522] implement load uuid_resolver --- .../src/index_controller/dump_actor/actor.rs | 68 +++---- .../dump_actor/handle_impl.rs | 6 +- .../dump_actor/loaders/mod.rs | 2 + .../index_controller/dump_actor/loaders/v1.rs | 137 ++++++++++++++ .../index_controller/dump_actor/loaders/v2.rs | 179 ++++++++++++++++++ .../src/index_controller/dump_actor/mod.rs | 148 +++------------ .../src/index_controller/dump_actor/v1.rs | 122 ------------ .../src/index_controller/dump_actor/v2.rs | 89 --------- meilisearch-http/src/index_controller/mod.rs | 30 +-- .../src/index_controller/update_actor/mod.rs | 2 +- .../update_actor/store/dump.rs | 2 +- .../update_actor/store/mod.rs | 6 +- .../src/index_controller/uuid_resolver/mod.rs | 2 +- .../index_controller/uuid_resolver/store.rs | 56 +++++- 14 files changed, 438 insertions(+), 411 deletions(-) create mode 100644 meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs create mode 100644 meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs create mode 100644 meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs delete mode 100644 meilisearch-http/src/index_controller/dump_actor/v1.rs delete mode 100644 meilisearch-http/src/index_controller/dump_actor/v2.rs diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 2d931dcbd..31378f89c 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -1,27 +1,26 @@ use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus}; use crate::helpers::compression; -use crate::index_controller::{index_actor, update_actor, uuid_resolver, IndexMetadata}; +use crate::index_controller::{update_actor, uuid_resolver}; use async_stream::stream; use chrono::Utc; use futures::stream::StreamExt; use log::{error, info}; use std::{ - collections::HashSet, path::{Path, PathBuf}, sync::Arc, }; -use tokio::sync::{mpsc, oneshot, RwLock}; -use uuid::Uuid; +use tokio::{fs::create_dir_all, sync::{mpsc, oneshot, RwLock}}; pub const CONCURRENT_DUMP_MSG: usize = 10; -pub struct DumpActor { +pub struct DumpActor { inbox: Option>, uuid_resolver: UuidResolver, - index: Index, update: Update, dump_path: PathBuf, dump_info: Arc>>, + _update_db_size: u64, + _index_db_size: u64, } /// Generate uid from creation date @@ -29,26 +28,27 @@ fn generate_uid() -> String { Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() } -impl DumpActor +impl DumpActor where UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, - Index: index_actor::IndexActorHandle + Send + Sync + Clone + 'static, Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, { pub fn new( inbox: mpsc::Receiver, uuid_resolver: UuidResolver, - index: Index, update: Update, dump_path: impl AsRef, + _index_db_size: u64, + _update_db_size: u64, ) -> Self { Self { inbox: Some(inbox), uuid_resolver, - index, update, dump_path: dump_path.as_ref().into(), dump_info: Arc::new(RwLock::new(None)), + _index_db_size, + _update_db_size, } } @@ -155,7 +155,7 @@ where } async fn perform_dump( - dump_path: PathBuf, + path: PathBuf, uuid_resolver: UuidResolver, update_handle: Update, uid: String, @@ -166,19 +166,23 @@ where { info!("Performing dump."); - let dump_path_clone = dump_path.clone(); - let temp_dump_path = tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(dump_path_clone)).await??; + create_dir_all(&path).await?; - let uuids = uuid_resolver.dump(temp_dump_path.path().to_owned()).await?; + let path_clone = path.clone(); + let temp_dump_dir = tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); - update_handle.dump(uuids, temp_dump_path.path().to_owned()).await?; + let uuids = uuid_resolver.dump(temp_dump_path.clone()).await?; + + update_handle.dump(uuids, temp_dump_path.clone()).await?; - let dump_path = dump_path.join(format!("{}.dump", uid)); let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(&dump_path)?; - let temp_dump_file_path = temp_dump_file.path().to_owned(); - compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; + let temp_dump_file = tempfile::NamedTempFile::new_in(&path)?; + compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; + + let dump_path = path.join(format!("{}.dump", uid)); temp_dump_file.persist(&dump_path)?; + Ok(dump_path) }) .await??; @@ -187,29 +191,3 @@ where Ok(()) } - -async fn list_indexes( - uuid_resolver: &UuidResolver, - index: &Index, -) -> anyhow::Result> -where - UuidResolver: uuid_resolver::UuidResolverHandle, - Index: index_actor::IndexActorHandle, -{ - let uuids = uuid_resolver.list().await?; - - let mut ret = Vec::new(); - - for (uid, uuid) in uuids { - let meta = index.get_index_meta(uuid).await?; - let meta = IndexMetadata { - uuid, - name: uid.clone(), - uid, - meta, - }; - ret.push(meta); - } - - Ok(ret) -} diff --git a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs index 575119410..ff663798f 100644 --- a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs @@ -29,13 +29,15 @@ impl DumpActorHandleImpl { pub fn new( path: impl AsRef, uuid_resolver: crate::index_controller::uuid_resolver::UuidResolverHandleImpl, - index: crate::index_controller::index_actor::IndexActorHandleImpl, update: crate::index_controller::update_actor::UpdateActorHandleImpl, + index_db_size: u64, + update_db_size: u64, ) -> anyhow::Result { let (sender, receiver) = mpsc::channel(10); - let actor = DumpActor::new(receiver, uuid_resolver, index, update, path); + let actor = DumpActor::new(receiver, uuid_resolver, update, path, index_db_size, update_db_size); tokio::task::spawn(actor.run()); + Ok(Self { sender }) } } diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs new file mode 100644 index 000000000..ae6adc7cf --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs @@ -0,0 +1,2 @@ +pub mod v1; +pub mod v2; diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs new file mode 100644 index 000000000..76207ff7b --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -0,0 +1,137 @@ +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::index_controller::IndexMetadata; + +#[derive(Serialize, Deserialize, Debug)] +pub struct MetadataV1 { + db_version: String, + indexes: Vec, +} + +impl MetadataV1 { + pub fn load_dump(self, _src: impl AsRef, _dst: impl AsRef) -> anyhow::Result<()> { + todo!("implement load v1") + } +} + +// This is the settings used in the last version of meilisearch exporting dump in V1 +//#[derive(Default, Clone, Serialize, Deserialize, Debug)] +//#[serde(rename_all = "camelCase", deny_unknown_fields)] +//struct Settings { + //#[serde(default, deserialize_with = "deserialize_some")] + //pub ranking_rules: Option>>, + //#[serde(default, deserialize_with = "deserialize_some")] + //pub distinct_attribute: Option>, + //#[serde(default, deserialize_with = "deserialize_some")] + //pub searchable_attributes: Option>>, + //#[serde(default, deserialize_with = "deserialize_some")] + //pub displayed_attributes: Option>>, + //#[serde(default, deserialize_with = "deserialize_some")] + //pub stop_words: Option>>, + //#[serde(default, deserialize_with = "deserialize_some")] + //pub synonyms: Option>>>, + //#[serde(default, deserialize_with = "deserialize_some")] + //pub attributes_for_faceting: Option>>, +//} + +///// we need to **always** be able to convert the old settings to the settings currently being used +//impl From for index_controller::Settings { + //fn from(settings: Settings) -> Self { + //if settings.synonyms.flatten().is_some() { + //error!("`synonyms` are not yet implemented and thus will be ignored"); + //} + //Self { + //distinct_attribute: settings.distinct_attribute, + //// we need to convert the old `Vec` into a `BTreeSet` + //displayed_attributes: settings.displayed_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), + //searchable_attributes: settings.searchable_attributes, + //// we previously had a `Vec` but now we have a `HashMap` + //// representing the name of the faceted field + the type of the field. Since the type + //// was not known in the V1 of the dump we are just going to assume everything is a + //// String + //attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().map(|key| (key, String::from("string"))).collect())), + //// we need to convert the old `Vec` into a `BTreeSet` + //ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { + //match criterion.as_str() { + //"words" | "typo" | "proximity" | "attribute" => Some(criterion), + //s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), + //"wordsPosition" => { + //warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); + //Some(String::from("words")) + //} + //"exactness" => { + //error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); + //None + //} + //s => { + //error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); + //None + //} + //} + //}).collect())), + //// we need to convert the old `Vec` into a `BTreeSet` + //stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())), + //_kind: PhantomData, + //} + //} +//} + +///// Extract Settings from `settings.json` file present at provided `dir_path` +//fn import_settings(dir_path: &Path) -> anyhow::Result { + //let path = dir_path.join("settings.json"); + //let file = File::open(path)?; + //let reader = std::io::BufReader::new(file); + //let metadata = serde_json::from_reader(reader)?; + + //Ok(metadata) +//} + +//pub fn import_dump( + //size: usize, + //uuid: Uuid, + //dump_path: &Path, + //db_path: &Path, + //primary_key: Option<&str>, +//) -> anyhow::Result<()> { + //let index_path = db_path.join(&format!("indexes/index-{}", uuid)); + //info!("Importing a dump from an old version of meilisearch with dump version 1"); + + //std::fs::create_dir_all(&index_path)?; + //let mut options = EnvOpenOptions::new(); + //options.map_size(size); + //let index = milli::Index::new(options, index_path)?; + //let index = Index(Arc::new(index)); + + //// extract `settings.json` file and import content + //let settings = import_settings(&dump_path)?; + //let settings: index_controller::Settings = settings.into(); + //let update_builder = UpdateBuilder::new(0); + //index.update_settings(&settings.check(), update_builder)?; + + //let update_builder = UpdateBuilder::new(1); + //let file = File::open(&dump_path.join("documents.jsonl"))?; + //let reader = std::io::BufReader::new(file); + + //// TODO: TAMO: waiting for milli. We should use the result + //let _ = index.update_documents( + //UpdateFormat::JsonStream, + //IndexDocumentsMethod::ReplaceDocuments, + //Some(reader), + //update_builder, + //primary_key, + //); + + //// the last step: we extract the original milli::Index and close it + //Arc::try_unwrap(index.0) + //.map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") + //.unwrap() + //.prepare_for_closing() + //.wait(); + + //// at this point we should handle the import of the updates, but since the update logic is not handled in + //// meilisearch we are just going to ignore this part + + //Ok(()) +//} diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs new file mode 100644 index 000000000..ee7044fd1 --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -0,0 +1,179 @@ +use std::{fs::File, io::BufReader, marker::PhantomData, path::Path}; + +use anyhow::Context; +use chrono::{DateTime, Utc}; +use log::info; +use serde::{Deserialize, Serialize}; + +use crate::index_controller::uuid_resolver::store::UuidStore; + +#[derive(Serialize, Deserialize, Debug)] +pub struct MetadataV2 { + db_version: String, + index_db_size: usize, + update_db_size: usize, + dump_date: DateTime, + _pth: PhantomData, +} + +impl MetadataV2 +where U: UuidStore, +{ + pub fn load_dump(self, src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + info!( + "Loading dump from {}, dump database version: {}, dump version: V2", + self.dump_date, self.db_version + ); + // get dir in which to load the db: + let dst_dir = dst + .as_ref() + .parent() + .with_context(|| format!("Invalid db path: {}", dst.as_ref().display()))?; + + let tmp_dst = tempfile::tempdir_in(dst_dir)?; + + self.load_index_resolver(&src, tmp_dst.path())?; + load_updates(&src, tmp_dst.path())?; + load_indexes(&src, tmp_dst.path())?; + Ok(()) + } + + fn load_index_resolver( + &self, + src: impl AsRef, + dst: impl AsRef, + ) -> anyhow::Result<()> { + info!("Loading index database."); + let uuid_resolver_path = dst.as_ref().join("uuid_resolver/"); + std::fs::create_dir_all(&uuid_resolver_path)?; + + U::load_dump(src.as_ref(), dst.as_ref())?; + + Ok(()) + } +} + + +fn load_updates(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + info!("Loading updates."); + todo!() +} + +fn load_indexes(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + info!("Loading indexes"); + todo!() +} + +// Extract Settings from `settings.json` file present at provided `dir_path` +//fn import_settings(dir_path: &Path) -> anyhow::Result> { +//let path = dir_path.join("settings.json"); +//let file = File::open(path)?; +//let reader = BufReader::new(file); +//let metadata: Settings = serde_json::from_reader(reader)?; + +//Ok(metadata.check()) +//} + +//pub fn import_dump( +//_db_size: usize, +//update_db_size: usize, +//_uuid: Uuid, +//dump_path: impl AsRef, +//db_path: impl AsRef, +//_primary_key: Option<&str>, +//) -> anyhow::Result<()> { +//info!("Dump import started."); +//info!("Importing outstanding updates..."); + +//import_updates(&dump_path, &db_path, update_db_size)?; + +//info!("done importing updates"); + +//Ok(()) +////let index_path = db_path.join(&format!("indexes/index-{}", uuid)); +////std::fs::create_dir_all(&index_path)?; +////let mut options = EnvOpenOptions::new(); +////options.map_size(size); +////let index = milli::Index::new(options, index_path)?; +////let index = Index(Arc::new(index)); + +////let mut txn = index.write_txn()?; + +////info!("importing the settings..."); +////// extract `settings.json` file and import content +////let settings = import_settings(&dump_path)?; +////let update_builder = UpdateBuilder::new(0); +////index.update_settings_txn(&mut txn, &settings, update_builder)?; + +////// import the documents in the index +////let update_builder = UpdateBuilder::new(1); +////let file = File::open(&dump_path.join("documents.jsonl"))?; +////let reader = std::io::BufReader::new(file); + +////info!("importing the documents..."); +////// TODO: TAMO: currently we ignore any error caused by the importation of the documents because +////// if there is no documents nor primary key it'll throw an anyhow error, but we must remove +////// this before the merge on main +////index.update_documents_txn( +////&mut txn, +////UpdateFormat::JsonStream, +////IndexDocumentsMethod::ReplaceDocuments, +////Some(reader), +////update_builder, +////primary_key, +////)?; + +////txn.commit()?; + +////// the last step: we extract the original milli::Index and close it +////Arc::try_unwrap(index.0) +////.map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") +////.unwrap() +////.prepare_for_closing() +////.wait(); + +////info!("importing the updates..."); +////import_updates(dump_path, db_path) +//} + +//fn import_updates( +//src_path: impl AsRef, +//dst_path: impl AsRef, +//_update_db_size: usize +//) -> anyhow::Result<()> { +//let dst_update_path = dst_path.as_ref().join("updates"); +//std::fs::create_dir_all(&dst_update_path)?; + +//let dst_update_files_path = dst_update_path.join("update_files"); +//std::fs::create_dir_all(&dst_update_files_path)?; + +//let options = EnvOpenOptions::new(); +//let (update_store, _) = UpdateStore::create(options, &dst_update_path)?; + +//let src_update_path = src_path.as_ref().join("updates"); +//let src_update_files_path = src_update_path.join("update_files"); +//let update_data = File::open(&src_update_path.join("data.jsonl"))?; +//let mut update_data = BufReader::new(update_data); + +//let mut wtxn = update_store.env.write_txn()?; +//let mut line = String::new(); +//loop { +//match update_data.read_line(&mut line) { +//Ok(_) => { +//let UpdateEntry { uuid, mut update } = serde_json::from_str(&line)?; + +//if let Some(path) = update.content_path_mut() { +//let dst_file_path = dst_update_files_path.join(&path); +//let src_file_path = src_update_files_path.join(&path); +//*path = dst_update_files_path.join(&path); +//std::fs::copy(src_file_path, dst_file_path)?; +//} + +//update_store.register_raw_updates(&mut wtxn, update, uuid)?; +//} +//_ => break, +//} +//} +//wtxn.commit()?; +//Ok(()) +//} diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 1508f8eb7..f0eeb1be3 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,26 +1,18 @@ mod actor; mod handle_impl; mod message; -mod v1; -mod v2; +mod loaders; -use std::{fs::File, path::Path, sync::Arc}; +use std::{fs::File, path::Path}; -use anyhow::bail; -use heed::EnvOpenOptions; -use log::{error, info}; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use log::error; #[cfg(test)] use mockall::automock; use serde::{Deserialize, Serialize}; -use tempfile::TempDir; use thiserror::Error; -use uuid::Uuid; -use super::IndexMetadata; -use crate::helpers::compression; -use crate::index::Index; -use crate::index_controller::uuid_resolver; +use loaders::v1::MetadataV1; +use loaders::v2::MetadataV2; pub use actor::DumpActor; pub use handle_impl::*; @@ -40,31 +32,6 @@ pub enum DumpError { DumpDoesNotExist(String), } -#[derive(Debug, Serialize, Deserialize, Copy, Clone)] -enum DumpVersion { - V1, - V2, -} - -impl DumpVersion { - const CURRENT: Self = Self::V2; - - /// Select the good importation function from the `DumpVersion` of metadata - pub fn import_index( - self, - size: usize, - uuid: Uuid, - dump_path: &Path, - db_path: &Path, - primary_key: Option<&str>, - ) -> anyhow::Result<()> { - match self { - Self::V1 => v1::import_index(size, uuid, dump_path, db_path, primary_key), - Self::V2 => v2::import_index(size, uuid, dump_path, db_path, primary_key), - } - } -} - #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait DumpActorHandle { @@ -78,23 +45,19 @@ pub trait DumpActorHandle { } #[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Metadata { - indexes: Vec, - db_version: String, - dump_version: DumpVersion, +#[serde(rename_all = "camelCase", tag = "dump_version")] +pub enum Metadata { + V1 { + #[serde(flatten)] + meta: MetadataV1, + }, + V2 { + #[serde(flatten)] + meta: MetadataV2, + }, } impl Metadata { - /// Create a Metadata with the current dump version of meilisearch. - pub fn new(indexes: Vec, db_version: String) -> Self { - Metadata { - indexes, - db_version, - dump_version: DumpVersion::CURRENT, - } - } - /// Extract Metadata from `metadata.json` file present at provided `dir_path` fn from_path(dir_path: &Path) -> anyhow::Result { let path = dir_path.join("metadata.json"); @@ -155,80 +118,19 @@ impl DumpInfo { } pub fn load_dump( - db_path: impl AsRef, - dump_path: impl AsRef, - size: usize, + dst_path: impl AsRef, + src_path: impl AsRef, + _index_db_size: u64, + _update_db_size: u64, ) -> anyhow::Result<()> { - info!("Importing dump from {}...", dump_path.as_ref().display()); - let db_path = db_path.as_ref(); - let dump_path = dump_path.as_ref(); - let uuid_resolver = uuid_resolver::HeedUuidStore::new(&db_path)?; + let meta_path = src_path.as_ref().join("metadat.json"); + let mut meta_file = File::open(&meta_path)?; + let meta: Metadata = serde_json::from_reader(&mut meta_file)?; - // extract the dump in a temporary directory - let tmp_dir = TempDir::new_in(db_path)?; - let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(dump_path, tmp_dir_path)?; - - // read dump metadata - let metadata = Metadata::from_path(&tmp_dir_path)?; - - // remove indexes which have same `uuid` than indexes to import and create empty indexes - let existing_index_uids = uuid_resolver.list()?; - - info!("Deleting indexes already present in the db and provided in the dump..."); - for idx in &metadata.indexes { - if let Some((_, uuid)) = existing_index_uids.iter().find(|(s, _)| s == &idx.uid) { - // if we find the index in the `uuid_resolver` it's supposed to exist on the file system - // and we want to delete it - let path = db_path.join(&format!("indexes/index-{}", uuid)); - info!("Deleting {}", path.display()); - use std::io::ErrorKind::*; - match std::fs::remove_dir_all(path) { - Ok(()) => (), - // if an index was present in the metadata but missing of the fs we can ignore the - // problem because we are going to create it later - Err(e) if e.kind() == NotFound => (), - Err(e) => bail!(e), - } - } else { - // if the index does not exist in the `uuid_resolver` we create it - uuid_resolver.create_uuid(idx.uid.clone(), false)?; - } + match meta { + Metadata::V1 { meta } => meta.load_dump(src_path, dst_path)?, + Metadata::V2 { meta } => meta.load_dump(src_path, dst_path)?, } - // import each indexes content - for idx in metadata.indexes { - let dump_path = tmp_dir_path.join(&idx.uid); - // this cannot fail since we created all the missing uuid in the previous loop - let uuid = uuid_resolver.get_uuid(idx.uid)?.unwrap(); - - info!( - "Importing dump from {} into {}...", - dump_path.display(), - db_path.display() - ); - metadata.dump_version.import_index( - size, - uuid, - &dump_path, - &db_path, - idx.meta.primary_key.as_ref().map(|s| s.as_ref()), - )?; - info!("Dump importation from {} succeed", dump_path.display()); - } - - // finally we can move all the unprocessed update file into our new DB - // this directory may not exists - let update_path = tmp_dir_path.join("update_files"); - let db_update_path = db_path.join("updates/update_files"); - if update_path.exists() { - let _ = std::fs::remove_dir_all(db_update_path); - std::fs::rename( - tmp_dir_path.join("update_files"), - db_path.join("updates/update_files"), - )?; - } - - info!("Dump importation from {} succeed", dump_path.display()); Ok(()) } diff --git a/meilisearch-http/src/index_controller/dump_actor/v1.rs b/meilisearch-http/src/index_controller/dump_actor/v1.rs deleted file mode 100644 index 6f199193c..000000000 --- a/meilisearch-http/src/index_controller/dump_actor/v1.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::{collections::{BTreeMap, BTreeSet}, marker::PhantomData}; - -use log::warn; -use serde::{Deserialize, Serialize}; -use crate::{index::Unchecked, index_controller}; -use crate::index::deserialize_some; -use super::*; - -/// This is the settings used in the last version of meilisearch exporting dump in V1 -#[derive(Default, Clone, Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct Settings { - #[serde(default, deserialize_with = "deserialize_some")] - pub ranking_rules: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub distinct_attribute: Option>, - #[serde(default, deserialize_with = "deserialize_some")] - pub searchable_attributes: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub displayed_attributes: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub stop_words: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub synonyms: Option>>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub attributes_for_faceting: Option>>, -} - -/// we need to **always** be able to convert the old settings to the settings currently being used -impl From for index_controller::Settings { - fn from(settings: Settings) -> Self { - if settings.synonyms.flatten().is_some() { - error!("`synonyms` are not yet implemented and thus will be ignored"); - } - Self { - distinct_attribute: settings.distinct_attribute, - // we need to convert the old `Vec` into a `BTreeSet` - displayed_attributes: settings.displayed_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), - searchable_attributes: settings.searchable_attributes, - // we previously had a `Vec` but now we have a `HashMap` - // representing the name of the faceted field + the type of the field. Since the type - // was not known in the V1 of the dump we are just going to assume everything is a - // String - attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().map(|key| (key, String::from("string"))).collect())), - // we need to convert the old `Vec` into a `BTreeSet` - ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { - match criterion.as_str() { - "words" | "typo" | "proximity" | "attribute" => Some(criterion), - s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), - "wordsPosition" => { - warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); - Some(String::from("words")) - } - "exactness" => { - error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); - None - } - s => { - error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); - None - } - } - }).collect())), - // we need to convert the old `Vec` into a `BTreeSet` - stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())), - _kind: PhantomData, - } - } -} - -/// Extract Settings from `settings.json` file present at provided `dir_path` -fn import_settings(dir_path: &Path) -> anyhow::Result { - let path = dir_path.join("settings.json"); - let file = File::open(path)?; - let reader = std::io::BufReader::new(file); - let metadata = serde_json::from_reader(reader)?; - - Ok(metadata) -} - - -pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { - let index_path = db_path.join(&format!("indexes/index-{}", uuid)); - info!("Importing a dump from an old version of meilisearch with dump version 1"); - - std::fs::create_dir_all(&index_path)?; - let mut options = EnvOpenOptions::new(); - options.map_size(size); - let index = milli::Index::new(options, index_path)?; - let index = Index(Arc::new(index)); - - // extract `settings.json` file and import content - let settings = import_settings(&dump_path)?; - let settings: index_controller::Settings = settings.into(); - let update_builder = UpdateBuilder::new(0); - index.update_settings(&settings.check(), update_builder)?; - - let update_builder = UpdateBuilder::new(1); - let file = File::open(&dump_path.join("documents.jsonl"))?; - let reader = std::io::BufReader::new(file); - - // TODO: TAMO: waiting for milli. We should use the result - let _ = index.update_documents( - UpdateFormat::JsonStream, - IndexDocumentsMethod::ReplaceDocuments, - Some(reader), - update_builder, - primary_key, - ); - - // the last step: we extract the original milli::Index and close it - Arc::try_unwrap(index.0) - .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") - .unwrap() - .prepare_for_closing() - .wait(); - - // at this point we should handle the import of the updates, but since the update logic is not handled in - // meilisearch we are just going to ignore this part - - Ok(()) -} diff --git a/meilisearch-http/src/index_controller/dump_actor/v2.rs b/meilisearch-http/src/index_controller/dump_actor/v2.rs deleted file mode 100644 index eeda78e8a..000000000 --- a/meilisearch-http/src/index_controller/dump_actor/v2.rs +++ /dev/null @@ -1,89 +0,0 @@ -use heed::EnvOpenOptions; -use log::info; -use uuid::Uuid; -use crate::{index::Unchecked, index_controller::{UpdateStatus, update_actor::UpdateStore}}; -use std::io::BufRead; -use milli::{update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}}; -use crate::index::{Checked, Index}; -use crate::index_controller::Settings; -use std::{fs::File, path::Path, sync::Arc}; - -/// Extract Settings from `settings.json` file present at provided `dir_path` -fn import_settings(dir_path: &Path) -> anyhow::Result> { - let path = dir_path.join("settings.json"); - let file = File::open(path)?; - let reader = std::io::BufReader::new(file); - let metadata: Settings = serde_json::from_reader(reader)?; - - println!("Meta: {:?}", metadata); - - Ok(metadata.check()) -} - -pub fn import_index(size: usize, uuid: Uuid, dump_path: &Path, db_path: &Path, primary_key: Option<&str>) -> anyhow::Result<()> { - let index_path = db_path.join(&format!("indexes/index-{}", uuid)); - std::fs::create_dir_all(&index_path)?; - let mut options = EnvOpenOptions::new(); - options.map_size(size); - let index = milli::Index::new(options, index_path)?; - let index = Index(Arc::new(index)); - - let mut txn = index.write_txn()?; - - info!("importing the settings..."); - // extract `settings.json` file and import content - let settings = import_settings(&dump_path)?; - let update_builder = UpdateBuilder::new(0); - index.update_settings_txn(&mut txn, &settings, update_builder)?; - - // import the documents in the index - let update_builder = UpdateBuilder::new(1); - let file = File::open(&dump_path.join("documents.jsonl"))?; - let reader = std::io::BufReader::new(file); - - info!("importing the documents..."); - // TODO: TAMO: currently we ignore any error caused by the importation of the documents because - // if there is no documents nor primary key it'll throw an anyhow error, but we must remove - // this before the merge on main - index.update_documents_txn( - &mut txn, - UpdateFormat::JsonStream, - IndexDocumentsMethod::ReplaceDocuments, - Some(reader), - update_builder, - primary_key, - )?; - - txn.commit()?; - - // the last step: we extract the original milli::Index and close it - Arc::try_unwrap(index.0) - .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") - .unwrap() - .prepare_for_closing() - .wait(); - - info!("importing the updates..."); - import_updates(uuid, dump_path, db_path) -} - -fn import_updates(uuid: Uuid, dump_path: &Path, db_path: &Path) -> anyhow::Result<()> { - let update_path = db_path.join("updates"); - let options = EnvOpenOptions::new(); - // create an UpdateStore to import the updates - std::fs::create_dir_all(&update_path)?; - let (update_store, _) = UpdateStore::create(options, &update_path)?; - let file = File::open(&dump_path.join("updates.jsonl"))?; - let reader = std::io::BufReader::new(file); - - let mut wtxn = update_store.env.write_txn()?; - for update in reader.lines() { - let mut update: UpdateStatus = serde_json::from_str(&update?)?; - if let Some(path) = update.content_path_mut() { - *path = update_path.join("update_files").join(&path); - } - update_store.register_raw_updates(&mut wtxn, update, uuid)?; - } - wtxn.commit()?; - Ok(()) -} diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 61bc71114..4e40a9873 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -14,22 +14,20 @@ use tokio::sync::mpsc; use tokio::time::sleep; use uuid::Uuid; -pub use updates::*; -pub use dump_actor::{DumpInfo, DumpStatus}; use dump_actor::DumpActorHandle; +pub use dump_actor::{DumpInfo, DumpStatus}; use index_actor::IndexActorHandle; -use snapshot::{SnapshotService, load_snapshot}; +use snapshot::{load_snapshot, SnapshotService}; use update_actor::UpdateActorHandle; +pub use updates::*; use uuid_resolver::{UuidResolverError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; -use dump_actor::load_dump; - +mod dump_actor; mod index_actor; mod snapshot; -mod dump_actor; mod update_actor; mod update_handler; mod updates; @@ -94,13 +92,8 @@ impl IndexController { options.ignore_snapshot_if_db_exists, options.ignore_missing_snapshot, )?; - } else if let Some(ref path) = options.import_dump { - load_dump( - &options.db_path, - path, - index_size, - )?; - + } else if let Some(ref _path) = options.import_dump { + todo!("implement load dump") } std::fs::create_dir_all(&path)?; @@ -112,7 +105,13 @@ impl IndexController { &path, update_store_size, )?; - let dump_handle = dump_actor::DumpActorHandleImpl::new(&options.dumps_dir, uuid_resolver.clone(), index_handle.clone(), update_handle.clone())?; + let dump_handle = dump_actor::DumpActorHandleImpl::new( + &options.dumps_dir, + uuid_resolver.clone(), + update_handle.clone(), + options.max_mdb_size.get_bytes(), + options.max_udb_size.get_bytes(), + )?; if options.schedule_snapshot { let snapshot_service = SnapshotService::new( @@ -158,7 +157,8 @@ impl IndexController { // prevent dead_locking between the update_handle::update that waits for the update to be // registered and the update_actor that waits for the the payload to be sent to it. tokio::task::spawn_local(async move { - payload.for_each(|r| async { + payload + .for_each(|r| async { let _ = sender.send(r).await; }) .await diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 8cd77e252..ba89eebe3 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -1,7 +1,7 @@ mod actor; mod handle_impl; mod message; -mod store; +pub mod store; use std::{collections::HashSet, path::PathBuf}; diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index 8b75f9e5d..82b8d0136 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -15,7 +15,7 @@ use super::UpdateStore; use crate::index_controller::{index_actor::IndexActorHandle, UpdateStatus}; #[derive(Serialize, Deserialize)] -struct UpdateEntry { +pub struct UpdateEntry { uuid: Uuid, update: UpdateStatus, } diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 52bd8d62a..58ac24720 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -1,4 +1,4 @@ -mod dump; +pub mod dump; mod codec; use std::collections::{BTreeMap, HashSet}; @@ -115,7 +115,6 @@ impl UpdateStore { let (notification_sender, notification_receiver) = mpsc::channel(10); // Send a first notification to trigger the process. - let _ = notification_sender.send(()); Ok(( Self { @@ -138,6 +137,9 @@ impl UpdateStore { let (update_store, mut notification_receiver) = Self::create(options, path)?; let update_store = Arc::new(update_store); + // trigger the update loop + let _ = update_store.notification_sender.send(()); + // Init update loop to perform any pending updates at launch. // Since we just launched the update store, and we still own the receiving end of the // channel, this call is guaranteed to succeed. diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index b84025094..5bddadf02 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -1,7 +1,7 @@ mod actor; mod handle_impl; mod message; -mod store; +pub mod store; use std::collections::HashSet; use std::path::PathBuf; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index b497116cb..0c6b66ddf 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, io::Write}; +use std::{collections::HashSet, io::{BufReader, BufRead, Write}}; use std::fs::{create_dir_all, File}; use std::path::{Path, PathBuf}; @@ -7,12 +7,19 @@ use heed::{ CompactionOption, Database, Env, EnvOpenOptions, }; use uuid::Uuid; +use serde::{Serialize, Deserialize}; use super::{Result, UuidResolverError, UUID_STORE_SIZE}; use crate::helpers::EnvSizer; +#[derive(Serialize, Deserialize)] +struct DumpEntry { + uuid: Uuid, + uid: String, +} + #[async_trait::async_trait] -pub trait UuidStore { +pub trait UuidStore: Sized { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return // the uuid otherwise. async fn create_uuid(&self, uid: String, err: bool) -> Result; @@ -23,6 +30,7 @@ pub trait UuidStore { async fn snapshot(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; async fn dump(&self, path: PathBuf) -> Result>; + fn load_dump(src: &Path, dst: &Path) -> Result<()>; } #[derive(Clone)] @@ -62,11 +70,7 @@ impl HeedUuidStore { Ok(uuid) } } - } - - pub fn get_uuid(&self, name: String) -> Result> { - let env = self.env.clone(); - let db = self.db; + } pub fn get_uuid(&self, name: String) -> Result> { let env = self.env.clone(); let db = self.db; let txn = env.read_txn()?; match db.get(&txn, &name)? { Some(uuid) => { @@ -149,11 +153,14 @@ impl HeedUuidStore { let txn = self.env.read_txn()?; for entry in self.db.iter(&txn)? { - let entry = entry?; + let (uid, uuid) = entry?; let uuid = Uuid::from_slice(entry.1)?; uuids.insert(uuid); - serde_json::to_writer(&mut dump_file, &serde_json::json!({ "uid": entry.0, "uuid": uuid - }))?; dump_file.write(b"\n").unwrap(); + let entry = DumpEntry { + uuid, uid + }; + serde_json::to_writer(&mut dump_file, &entry)?; + dump_file.write(b"\n").unwrap(); } Ok(uuids) @@ -200,4 +207,33 @@ impl UuidStore for HeedUuidStore { let this = self.clone(); tokio::task::spawn_blocking(move || this.dump(path)).await? } + + async fn load_dump(src: &Path, dst: &Path) -> Result<()> { + let uuid_resolver_path = dst.join("uuid_resolver/"); + std::fs::create_dir_all(&uuid_resolver_path)?; + + let src_indexes = src.join("index_uuids/data.jsonl"); + let indexes = File::Open(&src_indexes)?; + let mut indexes = BufReader::new(indexes); + let mut line = String::new(); + + let db = Self::new(dst)?; + let mut txn = db.env.write_txn()?; + + loop { + match indexes.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + let DumpEntry { uuid, uid } = serde_json::from_str(&line)?; + db.db.put(&mut txn, &uid, uuid.as_bytes())?; + } + Err(e) => Err(e)?, + } + + line.clear(); + } + txn.commit()?; + + Ok(()) + } } From b924e897f1096b0ce799c601ecdc5926e7a0424a Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 26 May 2021 22:52:06 +0200 Subject: [PATCH 351/522] load index dump --- meilisearch-http/src/index/dump.rs | 120 ++++++++++++ meilisearch-http/src/index/mod.rs | 69 ++----- .../update_handler.rs | 2 +- .../index_controller/dump_actor/loaders/v2.rs | 174 +++--------------- .../src/index_controller/dump_actor/mod.rs | 19 +- .../src/index_controller/index_actor/actor.rs | 4 +- .../src/index_controller/index_actor/store.rs | 14 +- meilisearch-http/src/index_controller/mod.rs | 1 - .../update_actor/store/dump.rs | 53 +++++- .../update_actor/store/mod.rs | 7 +- .../index_controller/uuid_resolver/store.rs | 77 ++++---- 11 files changed, 261 insertions(+), 279 deletions(-) create mode 100644 meilisearch-http/src/index/dump.rs rename meilisearch-http/src/{index_controller => index}/update_handler.rs (97%) diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs new file mode 100644 index 000000000..35f5159e5 --- /dev/null +++ b/meilisearch-http/src/index/dump.rs @@ -0,0 +1,120 @@ +use std::{fs::{create_dir_all, File}, path::Path, sync::Arc}; + +use anyhow::Context; +use heed::RoTxn; +use indexmap::IndexMap; +use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; +use serde::{Deserialize, Serialize}; +use anyhow::bail; + +use crate::option::IndexerOpts; + +use super::update_handler::UpdateHandler; +use super::{Checked, Index, Settings}; + +#[derive(Serialize, Deserialize)] +struct DumpMeta { + settings: Settings, + primary_key: Option, +} + +const META_FILE_NAME: &'static str = "meta.json"; +const DATA_FILE_NAME: &'static str = "documents.jsonl"; + +impl Index { + pub fn dump(&self, path: impl AsRef) -> anyhow::Result<()> { + // acquire write txn make sure any ongoing write is finnished before we start. + let txn = self.env.write_txn()?; + + self.dump_documents(&txn, &path)?; + self.dump_meta(&txn, &path)?; + + Ok(()) + } + + fn dump_documents(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { + println!("dumping documents"); + let document_file_path = path.as_ref().join(DATA_FILE_NAME); + let mut document_file = File::create(&document_file_path)?; + + let documents = self.all_documents(txn)?; + let fields_ids_map = self.fields_ids_map(txn)?; + + // dump documents + let mut json_map = IndexMap::new(); + for document in documents { + let (_, reader) = document?; + + for (fid, bytes) in reader.iter() { + if let Some(name) = fields_ids_map.name(fid) { + json_map.insert(name, serde_json::from_slice::(bytes)?); + } + } + + serde_json::to_writer(&mut document_file, &json_map)?; + std::io::Write::write(&mut document_file, b"\n")?; + + json_map.clear(); + } + + Ok(()) + } + + fn dump_meta(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { + println!("dumping settings"); + let meta_file_path = path.as_ref().join(META_FILE_NAME); + let mut meta_file = File::create(&meta_file_path)?; + + let settings = self.settings_txn(txn)?; + let primary_key = self.primary_key(txn)?.map(String::from); + let meta = DumpMeta { settings, primary_key }; + + serde_json::to_writer(&mut meta_file, &meta)?; + + Ok(()) + } + + pub fn load_dump( + src: impl AsRef, + dst: impl AsRef, + size: u64, + indexing_options: &IndexerOpts, + ) -> anyhow::Result<()> { + let dir_name = src + .as_ref() + .file_name() + .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?; + let dst_dir_path = dst.as_ref().join(dir_name); + create_dir_all(&dst_dir_path)?; + + let meta_path = src.as_ref().join(META_FILE_NAME); + let mut meta_file = File::open(meta_path)?; + let DumpMeta { settings, primary_key } = serde_json::from_reader(&mut meta_file)?; + let index = Self::open(&dst_dir_path, size as usize)?; + let mut txn = index.write_txn()?; + + let handler = UpdateHandler::new(&indexing_options)?; + + index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?; + + let document_file_path = src.as_ref().join(DATA_FILE_NAME); + let document_file = File::open(&document_file_path)?; + index.update_documents_txn( + &mut txn, + JsonStream, + IndexDocumentsMethod::UpdateDocuments, + Some(document_file), + handler.update_builder(0), + primary_key.as_deref(), + )?; + + txn.commit()?; + + match Arc::try_unwrap(index.0) { + Ok(inner) => inner.prepare_for_closing().wait(), + Err(_) => bail!("Could not close index properly."), + } + + Ok(()) + } +} diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index c4bf19856..331db07c4 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,11 +1,9 @@ -use std::{collections::{BTreeSet, HashSet}, io::Write, marker::PhantomData, path::{Path, PathBuf}}; +use std::{collections::{BTreeSet, HashSet}, marker::PhantomData, path::Path}; use std::ops::Deref; use std::sync::Arc; -use std::fs::File; use anyhow::{bail, Context}; -use heed::RoTxn; -use indexmap::IndexMap; +use heed::{EnvOpenOptions, RoTxn}; use milli::obkv_to_json; use serde_json::{Map, Value}; @@ -16,6 +14,8 @@ use serde::{de::Deserializer, Deserialize}; mod search; mod updates; +mod dump; +pub mod update_handler; pub type Document = Map; @@ -39,6 +39,14 @@ where } impl Index { + pub fn open(path: impl AsRef, size: usize) -> anyhow::Result { + std::fs::create_dir_all(&path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, &path)?; + Ok(Index(Arc::new(index))) + } + pub fn settings(&self) -> anyhow::Result> { let txn = self.read_txn()?; self.settings_txn(&txn) @@ -167,57 +175,4 @@ impl Index { displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid)); Ok(displayed_fields_ids) } - - pub fn dump(&self, path: PathBuf) -> anyhow::Result<()> { - // acquire write txn make sure any ongoing write is finnished before we start. - let txn = self.env.write_txn()?; - - self.dump_documents(&txn, &path)?; - self.dump_meta(&txn, &path)?; - - Ok(()) - } - - fn dump_documents(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { - println!("dumping documents"); - let document_file_path = path.as_ref().join("documents.jsonl"); - let mut document_file = File::create(&document_file_path)?; - - let documents = self.all_documents(txn)?; - let fields_ids_map = self.fields_ids_map(txn)?; - - // dump documents - let mut json_map = IndexMap::new(); - for document in documents { - let (_, reader) = document?; - - for (fid, bytes) in reader.iter() { - if let Some(name) = fields_ids_map.name(fid) { - json_map.insert(name, serde_json::from_slice::(bytes)?); - } - } - - serde_json::to_writer(&mut document_file, &json_map)?; - document_file.write(b"\n")?; - - json_map.clear(); - } - - Ok(()) - } - - fn dump_meta(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { - println!("dumping settings"); - let meta_file_path = path.as_ref().join("meta.json"); - let mut meta_file = File::create(&meta_file_path)?; - - let settings = self.settings_txn(txn)?; - let json = serde_json::json!({ - "settings": settings, - }); - - serde_json::to_writer(&mut meta_file, &json)?; - - Ok(()) - } } diff --git a/meilisearch-http/src/index_controller/update_handler.rs b/meilisearch-http/src/index/update_handler.rs similarity index 97% rename from meilisearch-http/src/index_controller/update_handler.rs rename to meilisearch-http/src/index/update_handler.rs index d0086aadd..6a303b4ce 100644 --- a/meilisearch-http/src/index_controller/update_handler.rs +++ b/meilisearch-http/src/index/update_handler.rs @@ -38,7 +38,7 @@ impl UpdateHandler { }) } - fn update_builder(&self, update_id: u64) -> UpdateBuilder { + pub fn update_builder(&self, update_id: u64) -> UpdateBuilder { // We prepare the update by using the update builder. let mut update_builder = UpdateBuilder::new(update_id); if let Some(max_nb_chunks) = self.max_nb_chunks { diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index ee7044fd1..ab4aa8cff 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -1,25 +1,27 @@ -use std::{fs::File, io::BufReader, marker::PhantomData, path::Path}; +use std::path::Path; use anyhow::Context; use chrono::{DateTime, Utc}; use log::info; use serde::{Deserialize, Serialize}; -use crate::index_controller::uuid_resolver::store::UuidStore; +use crate::{index::Index, index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}, option::IndexerOpts}; #[derive(Serialize, Deserialize, Debug)] -pub struct MetadataV2 { +pub struct MetadataV2 { db_version: String, - index_db_size: usize, - update_db_size: usize, + index_db_size: u64, + update_db_size: u64, dump_date: DateTime, - _pth: PhantomData, } -impl MetadataV2 -where U: UuidStore, -{ - pub fn load_dump(self, src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { +impl MetadataV2 { + pub fn load_dump( + self, + src: impl AsRef, + dst: impl AsRef, + indexing_options: &IndexerOpts, + ) -> anyhow::Result<()> { info!( "Loading dump from {}, dump database version: {}, dump version: V2", self.dump_date, self.db_version @@ -32,148 +34,26 @@ where U: UuidStore, let tmp_dst = tempfile::tempdir_in(dst_dir)?; - self.load_index_resolver(&src, tmp_dst.path())?; - load_updates(&src, tmp_dst.path())?; - load_indexes(&src, tmp_dst.path())?; - Ok(()) - } - - fn load_index_resolver( - &self, - src: impl AsRef, - dst: impl AsRef, - ) -> anyhow::Result<()> { info!("Loading index database."); let uuid_resolver_path = dst.as_ref().join("uuid_resolver/"); std::fs::create_dir_all(&uuid_resolver_path)?; + HeedUuidStore::load_dump(src.as_ref(), tmp_dst.as_ref())?; - U::load_dump(src.as_ref(), dst.as_ref())?; + info!("Loading updates."); + UpdateStore::load_dump(&src, &tmp_dst.as_ref(), self.update_db_size)?; + + info!("Loading indexes"); + let indexes_path = src.as_ref().join("indexes"); + let indexes = indexes_path.read_dir()?; + for index in indexes { + let index = index?; + Index::load_dump(&index.path(), &dst, self.index_db_size, indexing_options)?; + } + + // Persist and atomically rename the db + let persisted_dump = tmp_dst.into_path(); + std::fs::rename(&persisted_dump, &dst)?; Ok(()) } } - - -fn load_updates(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { - info!("Loading updates."); - todo!() -} - -fn load_indexes(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { - info!("Loading indexes"); - todo!() -} - -// Extract Settings from `settings.json` file present at provided `dir_path` -//fn import_settings(dir_path: &Path) -> anyhow::Result> { -//let path = dir_path.join("settings.json"); -//let file = File::open(path)?; -//let reader = BufReader::new(file); -//let metadata: Settings = serde_json::from_reader(reader)?; - -//Ok(metadata.check()) -//} - -//pub fn import_dump( -//_db_size: usize, -//update_db_size: usize, -//_uuid: Uuid, -//dump_path: impl AsRef, -//db_path: impl AsRef, -//_primary_key: Option<&str>, -//) -> anyhow::Result<()> { -//info!("Dump import started."); -//info!("Importing outstanding updates..."); - -//import_updates(&dump_path, &db_path, update_db_size)?; - -//info!("done importing updates"); - -//Ok(()) -////let index_path = db_path.join(&format!("indexes/index-{}", uuid)); -////std::fs::create_dir_all(&index_path)?; -////let mut options = EnvOpenOptions::new(); -////options.map_size(size); -////let index = milli::Index::new(options, index_path)?; -////let index = Index(Arc::new(index)); - -////let mut txn = index.write_txn()?; - -////info!("importing the settings..."); -////// extract `settings.json` file and import content -////let settings = import_settings(&dump_path)?; -////let update_builder = UpdateBuilder::new(0); -////index.update_settings_txn(&mut txn, &settings, update_builder)?; - -////// import the documents in the index -////let update_builder = UpdateBuilder::new(1); -////let file = File::open(&dump_path.join("documents.jsonl"))?; -////let reader = std::io::BufReader::new(file); - -////info!("importing the documents..."); -////// TODO: TAMO: currently we ignore any error caused by the importation of the documents because -////// if there is no documents nor primary key it'll throw an anyhow error, but we must remove -////// this before the merge on main -////index.update_documents_txn( -////&mut txn, -////UpdateFormat::JsonStream, -////IndexDocumentsMethod::ReplaceDocuments, -////Some(reader), -////update_builder, -////primary_key, -////)?; - -////txn.commit()?; - -////// the last step: we extract the original milli::Index and close it -////Arc::try_unwrap(index.0) -////.map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") -////.unwrap() -////.prepare_for_closing() -////.wait(); - -////info!("importing the updates..."); -////import_updates(dump_path, db_path) -//} - -//fn import_updates( -//src_path: impl AsRef, -//dst_path: impl AsRef, -//_update_db_size: usize -//) -> anyhow::Result<()> { -//let dst_update_path = dst_path.as_ref().join("updates"); -//std::fs::create_dir_all(&dst_update_path)?; - -//let dst_update_files_path = dst_update_path.join("update_files"); -//std::fs::create_dir_all(&dst_update_files_path)?; - -//let options = EnvOpenOptions::new(); -//let (update_store, _) = UpdateStore::create(options, &dst_update_path)?; - -//let src_update_path = src_path.as_ref().join("updates"); -//let src_update_files_path = src_update_path.join("update_files"); -//let update_data = File::open(&src_update_path.join("data.jsonl"))?; -//let mut update_data = BufReader::new(update_data); - -//let mut wtxn = update_store.env.write_txn()?; -//let mut line = String::new(); -//loop { -//match update_data.read_line(&mut line) { -//Ok(_) => { -//let UpdateEntry { uuid, mut update } = serde_json::from_str(&line)?; - -//if let Some(path) = update.content_path_mut() { -//let dst_file_path = dst_update_files_path.join(&path); -//let src_file_path = src_update_files_path.join(&path); -//*path = dst_update_files_path.join(&path); -//std::fs::copy(src_file_path, dst_file_path)?; -//} - -//update_store.register_raw_updates(&mut wtxn, update, uuid)?; -//} -//_ => break, -//} -//} -//wtxn.commit()?; -//Ok(()) -//} diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index f0eeb1be3..6d661d75c 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,8 +1,3 @@ -mod actor; -mod handle_impl; -mod message; -mod loaders; - use std::{fs::File, path::Path}; use log::error; @@ -18,6 +13,15 @@ pub use actor::DumpActor; pub use handle_impl::*; pub use message::DumpMsg; +use crate::option::IndexerOpts; + +use super::uuid_resolver::store::UuidStore; + +mod actor; +mod handle_impl; +mod loaders; +mod message; + pub type DumpResult = std::result::Result; #[derive(Error, Debug)] @@ -117,11 +121,12 @@ impl DumpInfo { } } -pub fn load_dump( +pub fn load_dump( dst_path: impl AsRef, src_path: impl AsRef, _index_db_size: u64, _update_db_size: u64, + indexer_opts: &IndexerOpts, ) -> anyhow::Result<()> { let meta_path = src_path.as_ref().join("metadat.json"); let mut meta_file = File::open(&meta_path)?; @@ -129,7 +134,7 @@ pub fn load_dump( match meta { Metadata::V1 { meta } => meta.load_dump(src_path, dst_path)?, - Metadata::V2 { meta } => meta.load_dump(src_path, dst_path)?, + Metadata::V2 { meta } => meta.load_dump(src_path.as_ref(), dst_path.as_ref(), indexer_opts)?, } Ok(()) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index f6f7cdc28..2f136c011 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -10,9 +10,9 @@ use tokio::{fs, sync::mpsc}; use tokio::task::spawn_blocking; use uuid::Uuid; -use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; +use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings, update_handler::UpdateHandler}; use crate::index_controller::{ - get_arc_ownership_blocking, update_handler::UpdateHandler, Failed, IndexStats, Processed, + get_arc_ownership_blocking, Failed, IndexStats, Processed, Processing, }; use crate::option::IndexerOpts; diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 3dee166a9..11791be48 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; -use heed::EnvOpenOptions; use tokio::fs; use tokio::sync::RwLock; use tokio::task::spawn_blocking; @@ -48,7 +47,7 @@ impl IndexStore for MapIndexStore { let index_size = self.index_size; let index = spawn_blocking(move || -> IndexResult { - let index = open_index(&path, index_size)?; + let index = Index::open(path, index_size)?; if let Some(primary_key) = primary_key { let mut txn = index.write_txn()?; index.put_primary_key(&mut txn, &primary_key)?; @@ -76,8 +75,7 @@ impl IndexStore for MapIndexStore { } let index_size = self.index_size; - let index = spawn_blocking(move || open_index(path, index_size)) - .await??; + let index = spawn_blocking(move || Index::open(path, index_size)).await??; self.index_store.write().await.insert(uuid, index.clone()); Ok(Some(index)) } @@ -91,11 +89,3 @@ impl IndexStore for MapIndexStore { Ok(index) } } - -fn open_index(path: impl AsRef, size: usize) -> IndexResult { - std::fs::create_dir_all(&path)?; - let mut options = EnvOpenOptions::new(); - options.map_size(size); - let index = milli::Index::new(options, &path)?; - Ok(Index(Arc::new(index))) -} diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 4e40a9873..69415a1cd 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -29,7 +29,6 @@ mod dump_actor; mod index_actor; mod snapshot; mod update_actor; -mod update_handler; mod updates; mod uuid_resolver; diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index 82b8d0136..1f36931d1 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -1,12 +1,7 @@ -use std::{ - collections::HashSet, - fs::{copy, create_dir_all, File}, - io::Write, - path::{Path, PathBuf}, -}; +use std::{collections::HashSet, fs::{copy, create_dir_all, File}, io::{BufRead, BufReader, Write}, path::{Path, PathBuf}}; use anyhow::Context; -use heed::RoTxn; +use heed::{EnvOpenOptions, RoTxn}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -15,7 +10,7 @@ use super::UpdateStore; use crate::index_controller::{index_actor::IndexActorHandle, UpdateStatus}; #[derive(Serialize, Deserialize)] -pub struct UpdateEntry { +struct UpdateEntry { uuid: Uuid, update: UpdateStatus, } @@ -121,6 +116,48 @@ impl UpdateStore { Ok(()) } + + pub fn load_dump(src: impl AsRef, dst: impl AsRef, db_size: u64) -> anyhow::Result<()> { + let dst_updates_path = dst.as_ref().join("updates/"); + create_dir_all(&dst_updates_path)?; + let dst_update_files_path = dst_updates_path.join("update_files/"); + create_dir_all(&dst_update_files_path)?; + + let mut options = EnvOpenOptions::new(); + options.map_size(db_size as usize); + let (store, _) = UpdateStore::new(options, &dst_updates_path)?; + + let src_update_path = src.as_ref().join("updates"); + let src_update_files_path = src_update_path.join("update_files"); + let update_data = File::open(&src_update_path.join("data.jsonl"))?; + let mut update_data = BufReader::new(update_data); + + let mut wtxn = store.env.write_txn()?; + let mut line = String::new(); + loop { + match update_data.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + let UpdateEntry { uuid, mut update } = serde_json::from_str(&line)?; + + if let Some(path) = update.content_path_mut() { + let dst_file_path = dst_update_files_path.join(&path); + let src_file_path = src_update_files_path.join(&path); + *path = dst_update_files_path.join(&path); + std::fs::copy(src_file_path, dst_file_path)?; + } + + store.register_raw_updates(&mut wtxn, update, uuid)?; + } + _ => break, + } + + line.clear(); + } + wtxn.commit()?; + + Ok(()) + } } async fn dump_indexes(uuids: &HashSet, handle: impl IndexActorHandle, path: impl AsRef)-> anyhow::Result<()> { diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 58ac24720..661b712ac 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -100,7 +100,7 @@ pub struct UpdateStore { } impl UpdateStore { - pub fn create( + fn new( mut options: EnvOpenOptions, path: impl AsRef, ) -> anyhow::Result<(Self, mpsc::Receiver<()>)> { @@ -114,7 +114,6 @@ impl UpdateStore { let state = Arc::new(StateLock::from_state(State::Idle)); let (notification_sender, notification_receiver) = mpsc::channel(10); - // Send a first notification to trigger the process. Ok(( Self { @@ -134,10 +133,10 @@ impl UpdateStore { path: impl AsRef, index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static, ) -> anyhow::Result> { - let (update_store, mut notification_receiver) = Self::create(options, path)?; + let (update_store, mut notification_receiver) = Self::new(options, path)?; let update_store = Arc::new(update_store); - // trigger the update loop + // Send a first notification to trigger the process. let _ = update_store.notification_sender.send(()); // Init update loop to perform any pending updates at launch. diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 0c6b66ddf..876c2454c 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -30,7 +30,6 @@ pub trait UuidStore: Sized { async fn snapshot(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; async fn dump(&self, path: PathBuf) -> Result>; - fn load_dump(src: &Path, dst: &Path) -> Result<()>; } #[derive(Clone)] @@ -46,14 +45,7 @@ impl HeedUuidStore { let mut options = EnvOpenOptions::new(); options.map_size(UUID_STORE_SIZE); // 1GB let env = options.open(path)?; - let db = env.create_database(None)?; - Ok(Self { env, db }) - } - - pub fn create_uuid(&self, name: String, err: bool) -> Result { - let env = self.env.clone(); - let db = self.db; - let mut txn = env.write_txn()?; + let db = env.create_database(None)?; Ok(Self { env, db }) } pub fn create_uuid(&self, name: String, err: bool) -> Result { let env = self.env.clone(); let db = self.db; let mut txn = env.write_txn()?; match db.get(&txn, &name)? { Some(uuid) => { if err { @@ -154,17 +146,51 @@ impl HeedUuidStore { let txn = self.env.read_txn()?; for entry in self.db.iter(&txn)? { let (uid, uuid) = entry?; - let uuid = Uuid::from_slice(entry.1)?; - uuids.insert(uuid); + let uid = uid.to_string(); + let uuid = Uuid::from_slice(uuid)?; + let entry = DumpEntry { uuid, uid }; serde_json::to_writer(&mut dump_file, &entry)?; dump_file.write(b"\n").unwrap(); + + uuids.insert(uuid); } Ok(uuids) } + + pub fn load_dump(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + let uuid_resolver_path = dst.as_ref().join("uuid_resolver/"); + std::fs::create_dir_all(&uuid_resolver_path)?; + + let src_indexes = src.as_ref().join("index_uuids/data.jsonl"); + let indexes = File::open(&src_indexes)?; + let mut indexes = BufReader::new(indexes); + let mut line = String::new(); + + let db = Self::new(dst)?; + let mut txn = db.env.write_txn()?; + + loop { + match indexes.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + let DumpEntry { uuid, uid } = serde_json::from_str(&line)?; + db.db.put(&mut txn, &uid, uuid.as_bytes())?; + } + Err(e) => Err(e)?, + } + + line.clear(); + } + txn.commit()?; + + db.env.prepare_for_closing().wait(); + + Ok(()) + } } #[async_trait::async_trait] @@ -207,33 +233,4 @@ impl UuidStore for HeedUuidStore { let this = self.clone(); tokio::task::spawn_blocking(move || this.dump(path)).await? } - - async fn load_dump(src: &Path, dst: &Path) -> Result<()> { - let uuid_resolver_path = dst.join("uuid_resolver/"); - std::fs::create_dir_all(&uuid_resolver_path)?; - - let src_indexes = src.join("index_uuids/data.jsonl"); - let indexes = File::Open(&src_indexes)?; - let mut indexes = BufReader::new(indexes); - let mut line = String::new(); - - let db = Self::new(dst)?; - let mut txn = db.env.write_txn()?; - - loop { - match indexes.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - let DumpEntry { uuid, uid } = serde_json::from_str(&line)?; - db.db.put(&mut txn, &uid, uuid.as_bytes())?; - } - Err(e) => Err(e)?, - } - - line.clear(); - } - txn.commit()?; - - Ok(()) - } } From c47369839bc73b4262b1aa79fb430cc462b65812 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 27 May 2021 10:51:19 +0200 Subject: [PATCH 352/522] dump meta --- .../src/index_controller/dump_actor/actor.rs | 123 ++++++++++-------- .../index_controller/dump_actor/loaders/v2.rs | 9 ++ .../src/index_controller/dump_actor/mod.rs | 4 + 3 files changed, 82 insertions(+), 54 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 31378f89c..1abceef47 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -1,17 +1,17 @@ use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus}; -use crate::helpers::compression; +use crate::{helpers::compression, index_controller::dump_actor::Metadata}; use crate::index_controller::{update_actor, uuid_resolver}; use async_stream::stream; use chrono::Utc; use futures::stream::StreamExt; use log::{error, info}; -use std::{ - path::{Path, PathBuf}, - sync::Arc, -}; +use update_actor::UpdateActorHandle; +use uuid_resolver::UuidResolverHandle; +use std::{fs::File, path::{Path, PathBuf}, sync::Arc}; use tokio::{fs::create_dir_all, sync::{mpsc, oneshot, RwLock}}; pub const CONCURRENT_DUMP_MSG: usize = 10; +const META_FILE_NAME: &'static str = "metadata.json"; pub struct DumpActor { inbox: Option>, @@ -19,8 +19,8 @@ pub struct DumpActor { update: Update, dump_path: PathBuf, dump_info: Arc>>, - _update_db_size: u64, - _index_db_size: u64, + update_db_size: u64, + index_db_size: u64, } /// Generate uid from creation date @@ -30,16 +30,16 @@ fn generate_uid() -> String { impl DumpActor where - UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, - Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, + UuidResolver: UuidResolverHandle + Send + Sync + Clone + 'static, + Update: UpdateActorHandle + Send + Sync + Clone + 'static, { pub fn new( inbox: mpsc::Receiver, uuid_resolver: UuidResolver, update: Update, dump_path: impl AsRef, - _index_db_size: u64, - _update_db_size: u64, + index_db_size: u64, + update_db_size: u64, ) -> Self { Self { inbox: Some(inbox), @@ -47,8 +47,8 @@ where update, dump_path: dump_path.as_ref().into(), dump_info: Arc::new(RwLock::new(None)), - _index_db_size, - _update_db_size, + index_db_size, + update_db_size, } } @@ -103,13 +103,16 @@ where let dump_info = self.dump_info.clone(); - let task_result = tokio::task::spawn(perform_dump( - self.dump_path.clone(), - self.uuid_resolver.clone(), - self.update.clone(), - uid.clone(), - )) - .await; + let task = DumpTask { + path: self.dump_path.clone(), + uuid_resolver: self.uuid_resolver.clone(), + update_handle: self.update.clone(), + uid: uid.clone(), + update_db_size: self.update_db_size, + index_db_size: self.index_db_size, + }; + + let task_result = tokio::task::spawn(task.run()).await; match task_result { Ok(Ok(())) => { @@ -152,42 +155,54 @@ where }) ) } + } -async fn perform_dump( +struct DumpTask { path: PathBuf, - uuid_resolver: UuidResolver, - update_handle: Update, + uuid_resolver: U, + update_handle: P, uid: String, -) -> anyhow::Result<()> -where - UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, - Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, -{ - info!("Performing dump."); - - create_dir_all(&path).await?; - - let path_clone = path.clone(); - let temp_dump_dir = tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); - - let uuids = uuid_resolver.dump(temp_dump_path.clone()).await?; - - update_handle.dump(uuids, temp_dump_path.clone()).await?; - - let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(&path)?; - compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; - - let dump_path = path.join(format!("{}.dump", uid)); - temp_dump_file.persist(&dump_path)?; - - Ok(dump_path) - }) - .await??; - - info!("Created dump in {:?}.", dump_path); - - Ok(()) + update_db_size: u64, + index_db_size: u64, +} + +impl DumpTask +where + U: UuidResolverHandle + Send + Sync + Clone + 'static, + P: UpdateActorHandle + Send + Sync + Clone + 'static, +{ + async fn run(self) -> anyhow::Result<()> { + info!("Performing dump."); + + create_dir_all(&self.path).await?; + + let path_clone = self.path.clone(); + let temp_dump_dir = tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let meta = Metadata::new_v2(self.index_db_size, self.update_db_size); + let meta_path = temp_dump_path.join(META_FILE_NAME); + let mut meta_file = File::create(&meta_path)?; + serde_json::to_writer(&mut meta_file, &meta)?; + + let uuids = self.uuid_resolver.dump(temp_dump_path.clone()).await?; + + self.update_handle.dump(uuids, temp_dump_path.clone()).await?; + + let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; + compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; + + let dump_path = self.path.join(format!("{}.dump", self.uid)); + temp_dump_file.persist(&dump_path)?; + + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(()) + } } diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index ab4aa8cff..b9f89ebbf 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -16,6 +16,15 @@ pub struct MetadataV2 { } impl MetadataV2 { + pub fn new(index_db_size: u64, update_db_size: u64) -> Self { + Self { + db_version: env!("CARGO_PKG_VERSION").to_string(), + index_db_size, + update_db_size, + dump_date: Utc::now(), + } + } + pub fn load_dump( self, src: impl AsRef, diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 6d661d75c..b54783f75 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -62,6 +62,10 @@ pub enum Metadata { } impl Metadata { + pub fn new_v2(index_db_size: u64, update_db_size: u64) -> Self { + let meta = MetadataV2::new(index_db_size, update_db_size); + Self::V2 { meta } + } /// Extract Metadata from `metadata.json` file present at provided `dir_path` fn from_path(dir_path: &Path) -> anyhow::Result { let path = dir_path.join("metadata.json"); From b258f4f394270ba3bc998a7f13d42312cae4675b Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 27 May 2021 14:30:20 +0200 Subject: [PATCH 353/522] fix dump import --- meilisearch-http/src/index/dump.rs | 12 +- meilisearch-http/src/index/updates.rs | 22 ++++ .../src/index_controller/dump_actor/actor.rs | 63 ++--------- .../index_controller/dump_actor/loaders/v2.rs | 17 ++- .../src/index_controller/dump_actor/mod.rs | 105 +++++++++++++----- meilisearch-http/src/index_controller/mod.rs | 12 +- .../index_controller/uuid_resolver/store.rs | 1 + 7 files changed, 133 insertions(+), 99 deletions(-) diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 35f5159e5..9dbb14fbd 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -9,12 +9,11 @@ use anyhow::bail; use crate::option::IndexerOpts; -use super::update_handler::UpdateHandler; -use super::{Checked, Index, Settings}; +use super::{Unchecked, Index, Settings, update_handler::UpdateHandler}; #[derive(Serialize, Deserialize)] struct DumpMeta { - settings: Settings, + settings: Settings, primary_key: Option, } @@ -33,7 +32,6 @@ impl Index { } fn dump_documents(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { - println!("dumping documents"); let document_file_path = path.as_ref().join(DATA_FILE_NAME); let mut document_file = File::create(&document_file_path)?; @@ -61,11 +59,10 @@ impl Index { } fn dump_meta(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { - println!("dumping settings"); let meta_file_path = path.as_ref().join(META_FILE_NAME); let mut meta_file = File::create(&meta_file_path)?; - let settings = self.settings_txn(txn)?; + let settings = self.settings_txn(txn)?.into_unchecked(); let primary_key = self.primary_key(txn)?.map(String::from); let meta = DumpMeta { settings, primary_key }; @@ -84,12 +81,13 @@ impl Index { .as_ref() .file_name() .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?; - let dst_dir_path = dst.as_ref().join(dir_name); + let dst_dir_path = dst.as_ref().join("indexes").join(dir_name); create_dir_all(&dst_dir_path)?; let meta_path = src.as_ref().join(META_FILE_NAME); let mut meta_file = File::open(meta_path)?; let DumpMeta { settings, primary_key } = serde_json::from_reader(&mut meta_file)?; + let settings = settings.check(); let index = Self::open(&dst_dir_path, size as usize)?; let mut txn = index.write_txn()?; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 2b489451b..053ca6a60 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -87,6 +87,28 @@ impl Settings { _kind: PhantomData, } } + + pub fn into_unchecked(self) -> Settings { + let Self { + displayed_attributes, + searchable_attributes, + attributes_for_faceting, + ranking_rules, + stop_words, + distinct_attribute, + .. + } = self; + + Settings { + displayed_attributes, + searchable_attributes, + attributes_for_faceting, + ranking_rules, + stop_words, + distinct_attribute, + _kind: PhantomData, + } + } } impl Settings { diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 1abceef47..b93d6f42d 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -1,17 +1,18 @@ -use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus}; -use crate::{helpers::compression, index_controller::dump_actor::Metadata}; -use crate::index_controller::{update_actor, uuid_resolver}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + use async_stream::stream; use chrono::Utc; use futures::stream::StreamExt; use log::{error, info}; use update_actor::UpdateActorHandle; use uuid_resolver::UuidResolverHandle; -use std::{fs::File, path::{Path, PathBuf}, sync::Arc}; -use tokio::{fs::create_dir_all, sync::{mpsc, oneshot, RwLock}}; +use tokio::sync::{mpsc, oneshot, RwLock}; + +use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus, DumpTask}; +use crate::index_controller::{update_actor, uuid_resolver}; pub const CONCURRENT_DUMP_MSG: usize = 10; -const META_FILE_NAME: &'static str = "metadata.json"; pub struct DumpActor { inbox: Option>, @@ -155,54 +156,4 @@ where }) ) } - -} - -struct DumpTask { - path: PathBuf, - uuid_resolver: U, - update_handle: P, - uid: String, - update_db_size: u64, - index_db_size: u64, -} - -impl DumpTask -where - U: UuidResolverHandle + Send + Sync + Clone + 'static, - P: UpdateActorHandle + Send + Sync + Clone + 'static, -{ - async fn run(self) -> anyhow::Result<()> { - info!("Performing dump."); - - create_dir_all(&self.path).await?; - - let path_clone = self.path.clone(); - let temp_dump_dir = tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); - - let meta = Metadata::new_v2(self.index_db_size, self.update_db_size); - let meta_path = temp_dump_path.join(META_FILE_NAME); - let mut meta_file = File::create(&meta_path)?; - serde_json::to_writer(&mut meta_file, &meta)?; - - let uuids = self.uuid_resolver.dump(temp_dump_path.clone()).await?; - - self.update_handle.dump(uuids, temp_dump_path.clone()).await?; - - let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; - compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; - - let dump_path = self.path.join(format!("{}.dump", self.uid)); - temp_dump_file.persist(&dump_path)?; - - Ok(dump_path) - }) - .await??; - - info!("Created dump in {:?}.", dump_path); - - Ok(()) - } } diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index b9f89ebbf..def47fecb 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -2,7 +2,7 @@ use std::path::Path; use anyhow::Context; use chrono::{DateTime, Utc}; -use log::info; +use log::{info, warn}; use serde::{Deserialize, Serialize}; use crate::{index::Index, index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}, option::IndexerOpts}; @@ -29,6 +29,8 @@ impl MetadataV2 { self, src: impl AsRef, dst: impl AsRef, + _index_db_size: u64, + _update_db_size: u64, indexing_options: &IndexerOpts, ) -> anyhow::Result<()> { info!( @@ -44,23 +46,26 @@ impl MetadataV2 { let tmp_dst = tempfile::tempdir_in(dst_dir)?; info!("Loading index database."); - let uuid_resolver_path = dst.as_ref().join("uuid_resolver/"); - std::fs::create_dir_all(&uuid_resolver_path)?; - HeedUuidStore::load_dump(src.as_ref(), tmp_dst.as_ref())?; + HeedUuidStore::load_dump(src.as_ref(), &tmp_dst)?; info!("Loading updates."); - UpdateStore::load_dump(&src, &tmp_dst.as_ref(), self.update_db_size)?; + UpdateStore::load_dump(&src, &tmp_dst, self.update_db_size)?; info!("Loading indexes"); let indexes_path = src.as_ref().join("indexes"); let indexes = indexes_path.read_dir()?; for index in indexes { let index = index?; - Index::load_dump(&index.path(), &dst, self.index_db_size, indexing_options)?; + Index::load_dump(&index.path(), &tmp_dst, self.index_db_size, indexing_options)?; } // Persist and atomically rename the db let persisted_dump = tmp_dst.into_path(); + if dst.as_ref().exists() { + warn!("Overwriting database at {}", dst.as_ref().display()); + std::fs::remove_dir_all(&dst)?; + } + std::fs::rename(&persisted_dump, &dst)?; Ok(()) diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index b54783f75..2b7d8a3e0 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,6 +1,7 @@ -use std::{fs::File, path::Path}; +use std::fs::File; +use std::path::{Path, PathBuf}; -use log::error; +use log::{error, info}; #[cfg(test)] use mockall::automock; use serde::{Deserialize, Serialize}; @@ -12,16 +13,18 @@ use loaders::v2::MetadataV2; pub use actor::DumpActor; pub use handle_impl::*; pub use message::DumpMsg; +use tokio::fs::create_dir_all; -use crate::option::IndexerOpts; - -use super::uuid_resolver::store::UuidStore; +use super::{update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle}; +use crate::{helpers::compression, option::IndexerOpts}; mod actor; mod handle_impl; mod loaders; mod message; +const META_FILE_NAME: &'static str = "metadata.json"; + pub type DumpResult = std::result::Result; #[derive(Error, Debug)] @@ -66,23 +69,6 @@ impl Metadata { let meta = MetadataV2::new(index_db_size, update_db_size); Self::V2 { meta } } - /// Extract Metadata from `metadata.json` file present at provided `dir_path` - fn from_path(dir_path: &Path) -> anyhow::Result { - let path = dir_path.join("metadata.json"); - let file = File::open(path)?; - let reader = std::io::BufReader::new(file); - let metadata = serde_json::from_reader(reader)?; - - Ok(metadata) - } - - /// Write Metadata in `metadata.json` file at provided `dir_path` - pub async fn to_path(&self, dir_path: &Path) -> anyhow::Result<()> { - let path = dir_path.join("metadata.json"); - tokio::fs::write(path, serde_json::to_string(self)?).await?; - - Ok(()) - } } #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] @@ -125,21 +111,84 @@ impl DumpInfo { } } -pub fn load_dump( +pub fn load_dump( dst_path: impl AsRef, src_path: impl AsRef, - _index_db_size: u64, - _update_db_size: u64, + index_db_size: u64, + update_db_size: u64, indexer_opts: &IndexerOpts, ) -> anyhow::Result<()> { - let meta_path = src_path.as_ref().join("metadat.json"); + let tmp_src = tempfile::tempdir_in(".")?; + let tmp_src_path = tmp_src.path(); + + compression::from_tar_gz(&src_path, tmp_src_path)?; + + let meta_path = tmp_src_path.join(META_FILE_NAME); let mut meta_file = File::open(&meta_path)?; let meta: Metadata = serde_json::from_reader(&mut meta_file)?; match meta { - Metadata::V1 { meta } => meta.load_dump(src_path, dst_path)?, - Metadata::V2 { meta } => meta.load_dump(src_path.as_ref(), dst_path.as_ref(), indexer_opts)?, + Metadata::V1 { meta } => meta.load_dump(&tmp_src_path, dst_path)?, + Metadata::V2 { meta } => meta.load_dump( + &tmp_src_path, + dst_path.as_ref(), + index_db_size, + update_db_size, + indexer_opts, + )?, } Ok(()) } + +struct DumpTask { + path: PathBuf, + uuid_resolver: U, + update_handle: P, + uid: String, + update_db_size: u64, + index_db_size: u64, +} + +impl DumpTask +where + U: UuidResolverHandle + Send + Sync + Clone + 'static, + P: UpdateActorHandle + Send + Sync + Clone + 'static, +{ + async fn run(self) -> anyhow::Result<()> { + info!("Performing dump."); + + create_dir_all(&self.path).await?; + + let path_clone = self.path.clone(); + let temp_dump_dir = + tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let meta = Metadata::new_v2(self.index_db_size, self.update_db_size); + let meta_path = temp_dump_path.join(META_FILE_NAME); + let mut meta_file = File::create(&meta_path)?; + serde_json::to_writer(&mut meta_file, &meta)?; + + let uuids = self.uuid_resolver.dump(temp_dump_path.clone()).await?; + + self.update_handle + .dump(uuids, temp_dump_path.clone()) + .await?; + + let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; + compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; + + let dump_path = self.path.join(format!("{}.dump", self.uid)); + temp_dump_file.persist(&dump_path)?; + + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(()) + } +} diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 69415a1cd..18ba6dee3 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -25,6 +25,8 @@ use uuid_resolver::{UuidResolverError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; +use self::dump_actor::load_dump; + mod dump_actor; mod index_actor; mod snapshot; @@ -91,8 +93,14 @@ impl IndexController { options.ignore_snapshot_if_db_exists, options.ignore_missing_snapshot, )?; - } else if let Some(ref _path) = options.import_dump { - todo!("implement load dump") + } else if let Some(ref src_path) = options.import_dump { + load_dump( + &options.db_path, + src_path, + options.max_mdb_size.get_bytes(), + options.max_udb_size.get_bytes(), + &options.indexer_options, + )?; } std::fs::create_dir_all(&path)?; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 876c2454c..2fd9ff301 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -178,6 +178,7 @@ impl HeedUuidStore { Ok(0) => break, Ok(_) => { let DumpEntry { uuid, uid } = serde_json::from_str(&line)?; + println!("importing {} {}", uid, uuid); db.db.put(&mut txn, &uid, uuid.as_bytes())?; } Err(e) => Err(e)?, From 1cb64caae46b4b5988f9f5c3fd8dec5d32dba068 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Sat, 29 May 2021 00:08:17 +0200 Subject: [PATCH 354/522] dump content is now only uuid --- .../index_controller/update_actor/actor.rs | 8 +- .../update_actor/store/dump.rs | 77 +++++++++---------- .../update_actor/store/mod.rs | 50 +++++++----- .../src/index_controller/updates.rs | 67 +--------------- 4 files changed, 75 insertions(+), 127 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 4097f31aa..40bba4e2b 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -117,7 +117,7 @@ where if file_len != 0 { file.flush().await?; let file = file.into_std().await; - Some((file, path)) + Some((file, update_file_id)) } else { // empty update, delete the empty file. fs::remove_file(&path).await?; @@ -133,7 +133,7 @@ where use std::io::{copy, sink, BufReader, Seek}; // If the payload is empty, ignore the check. - let path = if let Some((mut file, path)) = file_path { + let update_uuid = if let Some((mut file, uuid)) = file_path { // set the file back to the beginning file.seek(SeekFrom::Start(0))?; // Check that the json payload is valid: @@ -145,14 +145,14 @@ where file.seek(SeekFrom::Start(0))?; let _: serde_json::Value = serde_json::from_reader(file)?; } - Some(path) + Some(uuid) } else { None }; // The payload is valid, we can register it to the update store. let status = update_store - .register_update(meta, path, uuid) + .register_update(meta, update_uuid, uuid) .map(UpdateStatus::Enqueued)?; Ok(status) }) diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index 1f36931d1..ec7aeea87 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -1,12 +1,17 @@ -use std::{collections::HashSet, fs::{copy, create_dir_all, File}, io::{BufRead, BufReader, Write}, path::{Path, PathBuf}}; +use std::{ + collections::HashSet, + fs::{create_dir_all, File}, + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, +}; use anyhow::Context; use heed::{EnvOpenOptions, RoTxn}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use super::{State, codec::UpdateKeyCodec}; use super::UpdateStore; +use super::{codec::UpdateKeyCodec, State}; use crate::index_controller::{index_actor::IndexActorHandle, UpdateStatus}; #[derive(Serialize, Deserialize)] @@ -50,10 +55,10 @@ impl UpdateStore { let dump_data_path = path.as_ref().join("data.jsonl"); let mut dump_data_file = File::create(dump_data_path)?; - let update_files_path = path.as_ref().join("update_files"); + let update_files_path = path.as_ref().join(super::UPDATE_DIR); create_dir_all(&update_files_path)?; - self.dump_pending(&txn, uuids, &mut dump_data_file, &update_files_path)?; + self.dump_pending(&txn, uuids, &mut dump_data_file, &path)?; self.dump_completed(&txn, uuids, &mut dump_data_file)?; Ok(()) @@ -64,19 +69,24 @@ impl UpdateStore { txn: &RoTxn, uuids: &HashSet, mut file: &mut File, - update_files_path: impl AsRef, + dst_update_files: impl AsRef, ) -> anyhow::Result<()> { let pendings = self.pending_queue.iter(txn)?.lazily_decode_data(); for pending in pendings { let ((_, uuid, _), data) = pending?; if uuids.contains(&uuid) { - let mut update = data.decode()?; + let update = data.decode()?; - if let Some(content) = update.content.take() { - update.content = Some(dump_update_file(content, &update_files_path)?); + if let Some(ref update_uuid) = update.content { + let src = dbg!(super::update_uuid_to_file_path(&self.path, *update_uuid)); + let dst = dbg!(super::update_uuid_to_file_path(&dst_update_files, *update_uuid)); + assert!(src.exists()); + dbg!(std::fs::copy(src, dst))?; } + println!("copied files"); + let update_json = UpdateEntry { uuid, update: update.into(), @@ -117,18 +127,20 @@ impl UpdateStore { Ok(()) } - pub fn load_dump(src: impl AsRef, dst: impl AsRef, db_size: u64) -> anyhow::Result<()> { - let dst_updates_path = dst.as_ref().join("updates/"); - create_dir_all(&dst_updates_path)?; - let dst_update_files_path = dst_updates_path.join("update_files/"); - create_dir_all(&dst_update_files_path)?; + pub fn load_dump( + src: impl AsRef, + dst: impl AsRef, + db_size: u64, + ) -> anyhow::Result<()> { + let dst_update_path = dst.as_ref().join("updates/"); + create_dir_all(&dst_update_path)?; + let mut options = EnvOpenOptions::new(); options.map_size(db_size as usize); - let (store, _) = UpdateStore::new(options, &dst_updates_path)?; + let (store, _) = UpdateStore::new(options, &dst_update_path)?; let src_update_path = src.as_ref().join("updates"); - let src_update_files_path = src_update_path.join("update_files"); let update_data = File::open(&src_update_path.join("data.jsonl"))?; let mut update_data = BufReader::new(update_data); @@ -138,15 +150,7 @@ impl UpdateStore { match update_data.read_line(&mut line) { Ok(0) => break, Ok(_) => { - let UpdateEntry { uuid, mut update } = serde_json::from_str(&line)?; - - if let Some(path) = update.content_path_mut() { - let dst_file_path = dst_update_files_path.join(&path); - let src_file_path = src_update_files_path.join(&path); - *path = dst_update_files_path.join(&path); - std::fs::copy(src_file_path, dst_file_path)?; - } - + let UpdateEntry { uuid, update } = serde_json::from_str(&line)?; store.register_raw_updates(&mut wtxn, update, uuid)?; } _ => break, @@ -154,30 +158,25 @@ impl UpdateStore { line.clear(); } + + let dst_update_files_path = dst_update_path.join("update_files/"); + let src_update_files_path = src_update_path.join("update_files/"); + std::fs::copy(src_update_files_path, dst_update_files_path)?; + wtxn.commit()?; Ok(()) } } -async fn dump_indexes(uuids: &HashSet, handle: impl IndexActorHandle, path: impl AsRef)-> anyhow::Result<()> { +async fn dump_indexes( + uuids: &HashSet, + handle: impl IndexActorHandle, + path: impl AsRef, +) -> anyhow::Result<()> { for uuid in uuids { handle.dump(*uuid, path.as_ref().to_owned()).await?; } Ok(()) } - -fn dump_update_file( - file_path: impl AsRef, - dump_path: impl AsRef, -) -> anyhow::Result { - let filename: PathBuf = file_path - .as_ref() - .file_name() - .context("invalid update file name")? - .into(); - let dump_file_path = dump_path.as_ref().join(&filename); - copy(file_path, dump_file_path)?; - Ok(filename) -} diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 661b712ac..6910d5144 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -1,12 +1,11 @@ pub mod dump; mod codec; -use std::collections::{BTreeMap, HashSet}; +use std::{collections::{BTreeMap, HashSet}, path::PathBuf}; use std::fs::{copy, create_dir_all, remove_file, File}; use std::path::Path; use std::sync::Arc; -use anyhow::Context; use arc_swap::ArcSwap; use futures::StreamExt; use heed::types::{ByteSlice, OwnedType, SerdeJson}; @@ -27,6 +26,8 @@ use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, Ind #[allow(clippy::upper_case_acronyms)] type BEU64 = U64; +const UPDATE_DIR: &'static str = "update_files"; + pub struct UpdateStoreInfo { /// Size of the update store in bytes. pub size: u64, @@ -97,6 +98,7 @@ pub struct UpdateStore { pub state: Arc, /// Wake up the loop when a new event occurs. notification_sender: mpsc::Sender<()>, + path: PathBuf, } impl UpdateStore { @@ -106,7 +108,7 @@ impl UpdateStore { ) -> anyhow::Result<(Self, mpsc::Receiver<()>)> { options.max_dbs(5); - let env = options.open(path)?; + let env = options.open(&path)?; let pending_queue = env.create_database(Some("pending-queue"))?; let next_update_id = env.create_database(Some("next-update-id"))?; let updates = env.create_database(Some("updates"))?; @@ -123,6 +125,7 @@ impl UpdateStore { updates, state, notification_sender, + path: path.as_ref().to_owned(), }, notification_receiver, )) @@ -165,7 +168,7 @@ impl UpdateStore { match res { Ok(Some(_)) => (), Ok(None) => break, - Err(e) => error!("error while processing update: {}", e), + Err(e) => panic!("error while processing update: {}", e), } } // the ownership on the arc has been taken, we need to exit. @@ -217,13 +220,13 @@ impl UpdateStore { pub fn register_update( &self, meta: UpdateMeta, - content: Option>, + content: Option, index_uuid: Uuid, ) -> heed::Result { let mut txn = self.env.write_txn()?; let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?; - let meta = Enqueued::new(meta, update_id, content.map(|p| p.as_ref().to_owned())); + let meta = Enqueued::new(meta, update_id, content); self.pending_queue .put(&mut txn, &(global_id, index_uuid, update_id), &meta)?; @@ -290,9 +293,9 @@ impl UpdateStore { state.swap(State::Processing(index_uuid, processing.clone())); let file = match content_path { - Some(ref path) => { - let file = File::open(path) - .with_context(|| format!("file at path: {:?}", &content_path))?; + Some(uuid) => { + let path = update_uuid_to_file_path(&self.path, uuid); + let file = File::open(path)?; Some(file) } None => None, @@ -308,7 +311,8 @@ impl UpdateStore { self.pending_queue .delete(&mut wtxn, &(global_id, index_uuid, update_id))?; - if let Some(path) = content_path { + if let Some(uuid) = content_path { + let path = update_uuid_to_file_path(&self.path, uuid); remove_file(&path)?; } @@ -408,7 +412,7 @@ impl UpdateStore { pub fn delete_all(&self, index_uuid: Uuid) -> anyhow::Result<()> { let mut txn = self.env.write_txn()?; // Contains all the content file paths that we need to be removed if the deletion was successful. - let mut paths_to_remove = Vec::new(); + let mut uuids_to_remove = Vec::new(); let mut pendings = self.pending_queue.iter_mut(&mut txn)?.lazily_decode_data(); @@ -416,8 +420,8 @@ impl UpdateStore { if uuid == index_uuid { pendings.del_current()?; let mut pending = pending.decode()?; - if let Some(path) = pending.content.take() { - paths_to_remove.push(path); + if let Some(update_uuid) = pending.content.take() { + uuids_to_remove.push(update_uuid); } } } @@ -437,7 +441,9 @@ impl UpdateStore { txn.commit()?; - paths_to_remove.iter().for_each(|path| { + uuids_to_remove.iter() + .map(|uuid| update_uuid_to_file_path(&self.path, *uuid)) + .for_each(|path| { let _ = remove_file(path); }); @@ -468,7 +474,7 @@ impl UpdateStore { // create db snapshot self.env.copy_to_path(&db_path, CompactionOption::Enabled)?; - let update_files_path = update_path.join("update_files"); + let update_files_path = update_path.join(UPDATE_DIR); create_dir_all(&update_files_path)?; let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); @@ -476,10 +482,9 @@ impl UpdateStore { for entry in pendings { let ((_, uuid, _), pending) = entry?; if uuids.contains(&uuid) { - if let Some(path) = pending.decode()?.content_path() { - let name = path.file_name().unwrap(); - let to = update_files_path.join(name); - copy(path, to)?; + if let Enqueued { content: Some(uuid), .. } = pending.decode()? { + let path = update_uuid_to_file_path(&self.path, uuid); + copy(path, &update_files_path)?; } } } @@ -508,7 +513,8 @@ impl UpdateStore { let txn = self.env.read_txn()?; for entry in self.pending_queue.iter(&txn)? { let (_, pending) = entry?; - if let Some(path) = pending.content_path() { + if let Enqueued { content: Some(uuid), .. } = pending { + let path = update_uuid_to_file_path(&self.path, uuid); size += File::open(path)?.metadata()?.len(); } } @@ -521,6 +527,10 @@ impl UpdateStore { } } +fn update_uuid_to_file_path(root: impl AsRef, uuid: Uuid) -> PathBuf { + root.as_ref().join(UPDATE_DIR).join(format!("update_{}", uuid)) +} + #[cfg(test)] mod test { use super::*; diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 31f0005f8..0aacf9b6c 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -1,8 +1,7 @@ -use std::path::{Path, PathBuf}; - use chrono::{DateTime, Utc}; use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; +use uuid::Uuid; use crate::index::{Checked, Settings}; @@ -34,11 +33,11 @@ pub struct Enqueued { pub update_id: u64, pub meta: UpdateMeta, pub enqueued_at: DateTime, - pub content: Option, + pub content: Option, } impl Enqueued { - pub fn new(meta: UpdateMeta, update_id: u64, content: Option) -> Self { + pub fn new(meta: UpdateMeta, update_id: u64, content: Option) -> Self { Self { enqueued_at: Utc::now(), meta, @@ -68,14 +67,6 @@ impl Enqueued { pub fn id(&self) -> u64 { self.update_id } - - pub fn content_path(&self) -> Option<&Path> { - self.content.as_deref() - } - - pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { - self.content.as_mut() - } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -91,14 +82,6 @@ impl Processed { pub fn id(&self) -> u64 { self.from.id() } - - pub fn content_path(&self) -> Option<&Path> { - self.from.content_path() - } - - pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { - self.from.content_path_mut() - } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -118,14 +101,6 @@ impl Processing { self.from.meta() } - pub fn content_path(&self) -> Option<&Path> { - self.from.content_path() - } - - pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { - self.from.content_path_mut() - } - pub fn process(self, success: UpdateResult) -> Processed { Processed { success, @@ -155,14 +130,6 @@ impl Aborted { pub fn id(&self) -> u64 { self.from.id() } - - pub fn content_path(&self) -> Option<&Path> { - self.from.content_path() - } - - pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { - self.from.content_path_mut() - } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -178,14 +145,6 @@ impl Failed { pub fn id(&self) -> u64 { self.from.id() } - - pub fn content_path(&self) -> Option<&Path> { - self.from.content_path() - } - - pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { - self.from.content_path_mut() - } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -215,26 +174,6 @@ impl UpdateStatus { _ => None, } } - - pub fn content_path(&self) -> Option<&Path> { - match self { - UpdateStatus::Processing(u) => u.content_path(), - UpdateStatus::Processed(u) => u.content_path(), - UpdateStatus::Aborted(u) => u.content_path(), - UpdateStatus::Failed(u) => u.content_path(), - UpdateStatus::Enqueued(u) => u.content_path(), - } - } - - pub fn content_path_mut(&mut self) -> Option<&mut PathBuf> { - match self { - UpdateStatus::Processing(u) => u.content_path_mut(), - UpdateStatus::Processed(u) => u.content_path_mut(), - UpdateStatus::Aborted(u) => u.content_path_mut(), - UpdateStatus::Failed(u) => u.content_path_mut(), - UpdateStatus::Enqueued(u) => u.content_path_mut(), - } - } } impl From for UpdateStatus { From 39c16c0fe4c05241036fa952ff7c7d3e166d2de2 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Sun, 30 May 2021 12:35:17 +0200 Subject: [PATCH 355/522] fix dump import --- .../update_actor/store/dump.rs | 26 +++++++++++-------- .../update_actor/store/mod.rs | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index ec7aeea87..fb4b7d5ac 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -12,7 +12,7 @@ use uuid::Uuid; use super::UpdateStore; use super::{codec::UpdateKeyCodec, State}; -use crate::index_controller::{index_actor::IndexActorHandle, UpdateStatus}; +use crate::index_controller::{Enqueued, UpdateStatus, index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path}; #[derive(Serialize, Deserialize)] struct UpdateEntry { @@ -69,7 +69,7 @@ impl UpdateStore { txn: &RoTxn, uuids: &HashSet, mut file: &mut File, - dst_update_files: impl AsRef, + dst_path: impl AsRef, ) -> anyhow::Result<()> { let pendings = self.pending_queue.iter(txn)?.lazily_decode_data(); @@ -79,10 +79,9 @@ impl UpdateStore { let update = data.decode()?; if let Some(ref update_uuid) = update.content { - let src = dbg!(super::update_uuid_to_file_path(&self.path, *update_uuid)); - let dst = dbg!(super::update_uuid_to_file_path(&dst_update_files, *update_uuid)); - assert!(src.exists()); - dbg!(std::fs::copy(src, dst))?; + let src = super::update_uuid_to_file_path(&self.path, *update_uuid); + let dst = super::update_uuid_to_file_path(&dst_path, *update_uuid); + std::fs::copy(src, dst)?; } println!("copied files"); @@ -144,6 +143,8 @@ impl UpdateStore { let update_data = File::open(&src_update_path.join("data.jsonl"))?; let mut update_data = BufReader::new(update_data); + std::fs::create_dir_all(dst_update_path.join("update_files/"))?; + let mut wtxn = store.env.write_txn()?; let mut line = String::new(); loop { @@ -151,7 +152,14 @@ impl UpdateStore { Ok(0) => break, Ok(_) => { let UpdateEntry { uuid, update } = serde_json::from_str(&line)?; - store.register_raw_updates(&mut wtxn, update, uuid)?; + store.register_raw_updates(&mut wtxn, &update, uuid)?; + + // Copy ascociated update path if it exists + if let UpdateStatus::Enqueued(Enqueued { content: Some(uuid), .. }) = update { + let src = update_uuid_to_file_path(&src_update_path, uuid); + let dst = update_uuid_to_file_path(&dst_update_path, uuid); + std::fs::copy(src, dst)?; + } } _ => break, } @@ -159,10 +167,6 @@ impl UpdateStore { line.clear(); } - let dst_update_files_path = dst_update_path.join("update_files/"); - let src_update_files_path = src_update_path.join("update_files/"); - std::fs::copy(src_update_files_path, dst_update_files_path)?; - wtxn.commit()?; Ok(()) diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 6910d5144..6e8f87a79 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -244,7 +244,7 @@ impl UpdateStore { pub fn register_raw_updates( &self, wtxn: &mut heed::RwTxn, - update: UpdateStatus, + update: &UpdateStatus, index_uuid: Uuid, ) -> heed::Result<()> { match update { From 33c6c4f0eed2d7e8038ce54ac564e83ceb75c561 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Sun, 30 May 2021 15:55:17 +0200 Subject: [PATCH 356/522] add timestamos to dump info --- meilisearch-http/src/index/updates.rs | 5 +++-- meilisearch-http/src/index_controller/dump_actor/actor.rs | 4 +++- meilisearch-http/src/index_controller/dump_actor/mod.rs | 8 ++++++++ .../src/index_controller/update_actor/store/dump.rs | 3 --- .../src/index_controller/update_actor/store/mod.rs | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 053ca6a60..566356d5f 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -197,9 +197,10 @@ impl Index { builder.update_format(format); builder.index_documents_method(method); - let indexing_callback = - |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); + //let indexing_callback = + //|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); + let indexing_callback = |_, _| (); let gzipped = false; let addition = match content { diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index b93d6f42d..ff4c39f6d 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -97,7 +97,9 @@ where return; } let uid = generate_uid(); + let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress); + *self.dump_info.write().await = Some(info.clone()); ret.send(Ok(info)).expect("Dump actor is dead"); @@ -126,7 +128,7 @@ where } Err(_) => { error!("Dump panicked. Dump status set to failed"); - *dump_info.write().await = Some(DumpInfo::new(uid, DumpStatus::Failed)); + (*dump_info.write().await).as_mut().expect("Inconsistent dump service state").with_error("Unexpected error while performing dump.".to_string()); } }; } diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 2b7d8a3e0..b236122bd 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,6 +1,7 @@ use std::fs::File; use std::path::{Path, PathBuf}; +use chrono::{DateTime, Utc}; use log::{error, info}; #[cfg(test)] use mockall::automock; @@ -86,6 +87,9 @@ pub struct DumpInfo { pub status: DumpStatus, #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, + started_at: DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + finished_at: Option>, } impl DumpInfo { @@ -94,15 +98,19 @@ impl DumpInfo { uid, status, error: None, + started_at: Utc::now(), + finished_at: None, } } pub fn with_error(&mut self, error: String) { self.status = DumpStatus::Failed; + self.finished_at = Some(Utc::now()); self.error = Some(error); } pub fn done(&mut self) { + self.finished_at = Some(Utc::now()); self.status = DumpStatus::Done; } diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index fb4b7d5ac..fad8974f3 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -5,7 +5,6 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::Context; use heed::{EnvOpenOptions, RoTxn}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -84,8 +83,6 @@ impl UpdateStore { std::fs::copy(src, dst)?; } - println!("copied files"); - let update_json = UpdateEntry { uuid, update: update.into(), diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 6e8f87a79..29ccd4f34 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -168,7 +168,7 @@ impl UpdateStore { match res { Ok(Some(_)) => (), Ok(None) => break, - Err(e) => panic!("error while processing update: {}", e), + Err(e) => error!("error while processing update: {}", e), } } // the ownership on the arc has been taken, we need to exit. From bc5a5e37ea8127ffbfcd3dcec8d56cb9d28068f8 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 31 May 2021 10:42:31 +0200 Subject: [PATCH 357/522] fix dump v1 --- .../index_controller/dump_actor/loaders/v1.rs | 265 ++++++++++-------- .../index_controller/dump_actor/loaders/v2.rs | 27 +- .../src/index_controller/dump_actor/mod.rs | 38 ++- .../index_controller/uuid_resolver/store.rs | 25 +- 4 files changed, 201 insertions(+), 154 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 76207ff7b..471e66d17 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -1,137 +1,178 @@ -use std::path::Path; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs::File, + marker::PhantomData, + path::Path, + sync::Arc, +}; +use heed::EnvOpenOptions; +use log::{error, info, warn}; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; +use uuid::Uuid; -use crate::index_controller::IndexMetadata; +use crate::{index::deserialize_some, index_controller::uuid_resolver::HeedUuidStore}; +use crate::{ + index::{Index, Unchecked}, + index_controller::{self, IndexMetadata}, +}; #[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] pub struct MetadataV1 { db_version: String, indexes: Vec, } impl MetadataV1 { - pub fn load_dump(self, _src: impl AsRef, _dst: impl AsRef) -> anyhow::Result<()> { - todo!("implement load v1") + pub fn load_dump( + self, + src: impl AsRef, + dst: impl AsRef, + size: usize, + ) -> anyhow::Result<()> { + info!( + "Loading dump, dump database version: {}, dump version: V1", + self.db_version + ); + + dbg!("here"); + + let uuid_store = HeedUuidStore::new(&dst)?; + dbg!("here"); + for index in self.indexes { + let uuid = Uuid::new_v4(); + uuid_store.insert(index.uid.clone(), uuid)?; + let src = src.as_ref().join(index.uid); + load_index(&src, &dst, uuid, index.meta.primary_key.as_deref(), size)?; + } + + Ok(()) } } -// This is the settings used in the last version of meilisearch exporting dump in V1 -//#[derive(Default, Clone, Serialize, Deserialize, Debug)] -//#[serde(rename_all = "camelCase", deny_unknown_fields)] -//struct Settings { - //#[serde(default, deserialize_with = "deserialize_some")] - //pub ranking_rules: Option>>, - //#[serde(default, deserialize_with = "deserialize_some")] - //pub distinct_attribute: Option>, - //#[serde(default, deserialize_with = "deserialize_some")] - //pub searchable_attributes: Option>>, - //#[serde(default, deserialize_with = "deserialize_some")] - //pub displayed_attributes: Option>>, - //#[serde(default, deserialize_with = "deserialize_some")] - //pub stop_words: Option>>, - //#[serde(default, deserialize_with = "deserialize_some")] - //pub synonyms: Option>>>, - //#[serde(default, deserialize_with = "deserialize_some")] - //pub attributes_for_faceting: Option>>, -//} +//This is the settings used in the last version of meilisearch exporting dump in V1 +#[derive(Default, Clone, Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Settings { + #[serde(default, deserialize_with = "deserialize_some")] + pub ranking_rules: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub distinct_attribute: Option>, + #[serde(default, deserialize_with = "deserialize_some")] + pub searchable_attributes: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub displayed_attributes: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub stop_words: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub synonyms: Option>>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub attributes_for_faceting: Option>>, +} -///// we need to **always** be able to convert the old settings to the settings currently being used -//impl From for index_controller::Settings { - //fn from(settings: Settings) -> Self { - //if settings.synonyms.flatten().is_some() { - //error!("`synonyms` are not yet implemented and thus will be ignored"); - //} - //Self { - //distinct_attribute: settings.distinct_attribute, - //// we need to convert the old `Vec` into a `BTreeSet` - //displayed_attributes: settings.displayed_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), - //searchable_attributes: settings.searchable_attributes, - //// we previously had a `Vec` but now we have a `HashMap` - //// representing the name of the faceted field + the type of the field. Since the type - //// was not known in the V1 of the dump we are just going to assume everything is a - //// String - //attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().map(|key| (key, String::from("string"))).collect())), - //// we need to convert the old `Vec` into a `BTreeSet` - //ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { - //match criterion.as_str() { - //"words" | "typo" | "proximity" | "attribute" => Some(criterion), - //s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), - //"wordsPosition" => { - //warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); - //Some(String::from("words")) - //} - //"exactness" => { - //error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); - //None - //} - //s => { - //error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); - //None - //} - //} - //}).collect())), - //// we need to convert the old `Vec` into a `BTreeSet` - //stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())), - //_kind: PhantomData, - //} - //} -//} +impl std::ops::Deref for Settings { + type Target = Option>>; -///// Extract Settings from `settings.json` file present at provided `dir_path` -//fn import_settings(dir_path: &Path) -> anyhow::Result { - //let path = dir_path.join("settings.json"); - //let file = File::open(path)?; - //let reader = std::io::BufReader::new(file); - //let metadata = serde_json::from_reader(reader)?; + fn deref(&self) -> &Self::Target { + &self.stop_words + } +} - //Ok(metadata) -//} +fn load_index( + src: impl AsRef, + dst: impl AsRef, + uuid: Uuid, + primary_key: Option<&str>, + size: usize, +) -> anyhow::Result<()> { + let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid)); -//pub fn import_dump( - //size: usize, - //uuid: Uuid, - //dump_path: &Path, - //db_path: &Path, - //primary_key: Option<&str>, -//) -> anyhow::Result<()> { - //let index_path = db_path.join(&format!("indexes/index-{}", uuid)); - //info!("Importing a dump from an old version of meilisearch with dump version 1"); + std::fs::create_dir_all(&index_path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, index_path)?; + let index = Index(Arc::new(index)); - //std::fs::create_dir_all(&index_path)?; - //let mut options = EnvOpenOptions::new(); - //options.map_size(size); - //let index = milli::Index::new(options, index_path)?; - //let index = Index(Arc::new(index)); + // extract `settings.json` file and import content + let settings = import_settings(&src)?; + let settings: index_controller::Settings = settings.into(); + let update_builder = UpdateBuilder::new(0); + index.update_settings(&settings.check(), update_builder)?; - //// extract `settings.json` file and import content - //let settings = import_settings(&dump_path)?; - //let settings: index_controller::Settings = settings.into(); - //let update_builder = UpdateBuilder::new(0); - //index.update_settings(&settings.check(), update_builder)?; + let update_builder = UpdateBuilder::new(0); + let file = File::open(&src.as_ref().join("documents.jsonl"))?; + let reader = std::io::BufReader::new(file); - //let update_builder = UpdateBuilder::new(1); - //let file = File::open(&dump_path.join("documents.jsonl"))?; - //let reader = std::io::BufReader::new(file); + index.update_documents( + UpdateFormat::JsonStream, + IndexDocumentsMethod::ReplaceDocuments, + Some(reader), + update_builder, + primary_key, + )?; - //// TODO: TAMO: waiting for milli. We should use the result - //let _ = index.update_documents( - //UpdateFormat::JsonStream, - //IndexDocumentsMethod::ReplaceDocuments, - //Some(reader), - //update_builder, - //primary_key, - //); + // the last step: we extract the original milli::Index and close it + Arc::try_unwrap(index.0) + .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") + .unwrap() + .prepare_for_closing() + .wait(); - //// the last step: we extract the original milli::Index and close it - //Arc::try_unwrap(index.0) - //.map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") - //.unwrap() - //.prepare_for_closing() - //.wait(); + // Ignore updates in v1. - //// at this point we should handle the import of the updates, but since the update logic is not handled in - //// meilisearch we are just going to ignore this part + Ok(()) +} - //Ok(()) -//} +/// we need to **always** be able to convert the old settings to the settings currently being used +impl From for index_controller::Settings { + fn from(settings: Settings) -> Self { + if settings.synonyms.flatten().is_some() { + error!("`synonyms` are not yet implemented and thus will be ignored"); + } + Self { + distinct_attribute: settings.distinct_attribute, + // we need to convert the old `Vec` into a `BTreeSet` + displayed_attributes: settings.displayed_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), + searchable_attributes: settings.searchable_attributes, + // we previously had a `Vec` but now we have a `HashMap` + // representing the name of the faceted field + the type of the field. Since the type + // was not known in the V1 of the dump we are just going to assume everything is a + // String + attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().map(|key| (key, String::from("string"))).collect())), + // we need to convert the old `Vec` into a `BTreeSet` + ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { + match criterion.as_str() { + "words" | "typo" | "proximity" | "attribute" => Some(criterion), + s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), + "wordsPosition" => { + warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); + Some(String::from("words")) + } + "exactness" => { + error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); + None + } + s => { + error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); + None + } + } + }).collect())), + // we need to convert the old `Vec` into a `BTreeSet` + stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())), + _kind: PhantomData, + } + } +} + +/// Extract Settings from `settings.json` file present at provided `dir_path` +fn import_settings(dir_path: impl AsRef) -> anyhow::Result { + let path = dbg!(dir_path.as_ref().join("settings.json")); + let file = File::open(path)?; + let reader = std::io::BufReader::new(file); + let metadata = serde_json::from_reader(reader)?; + + Ok(metadata) +} diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index def47fecb..c0fe0abe6 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -1,13 +1,13 @@ use std::path::Path; -use anyhow::Context; use chrono::{DateTime, Utc}; -use log::{info, warn}; +use log::info; use serde::{Deserialize, Serialize}; use crate::{index::Index, index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}, option::IndexerOpts}; #[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] pub struct MetadataV2 { db_version: String, index_db_size: u64, @@ -29,6 +29,7 @@ impl MetadataV2 { self, src: impl AsRef, dst: impl AsRef, + // TODO: use these variable to test if loading the index is possible. _index_db_size: u64, _update_db_size: u64, indexing_options: &IndexerOpts, @@ -37,37 +38,21 @@ impl MetadataV2 { "Loading dump from {}, dump database version: {}, dump version: V2", self.dump_date, self.db_version ); - // get dir in which to load the db: - let dst_dir = dst - .as_ref() - .parent() - .with_context(|| format!("Invalid db path: {}", dst.as_ref().display()))?; - - let tmp_dst = tempfile::tempdir_in(dst_dir)?; info!("Loading index database."); - HeedUuidStore::load_dump(src.as_ref(), &tmp_dst)?; + HeedUuidStore::load_dump(src.as_ref(), &dst)?; info!("Loading updates."); - UpdateStore::load_dump(&src, &tmp_dst, self.update_db_size)?; + UpdateStore::load_dump(&src, &dst, self.update_db_size)?; info!("Loading indexes"); let indexes_path = src.as_ref().join("indexes"); let indexes = indexes_path.read_dir()?; for index in indexes { let index = index?; - Index::load_dump(&index.path(), &tmp_dst, self.index_db_size, indexing_options)?; + Index::load_dump(&index.path(), &dst, self.index_db_size, indexing_options)?; } - // Persist and atomically rename the db - let persisted_dump = tmp_dst.into_path(); - if dst.as_ref().exists() { - warn!("Overwriting database at {}", dst.as_ref().display()); - std::fs::remove_dir_all(&dst)?; - } - - std::fs::rename(&persisted_dump, &dst)?; - Ok(()) } } diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index b236122bd..e1998f876 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -2,11 +2,12 @@ use std::fs::File; use std::path::{Path, PathBuf}; use chrono::{DateTime, Utc}; -use log::{error, info}; +use log::{error, info, warn}; #[cfg(test)] use mockall::automock; use serde::{Deserialize, Serialize}; use thiserror::Error; +use anyhow::Context; use loaders::v1::MetadataV1; use loaders::v2::MetadataV2; @@ -53,22 +54,16 @@ pub trait DumpActorHandle { } #[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", tag = "dump_version")] +#[serde(tag = "dumpVersion")] pub enum Metadata { - V1 { - #[serde(flatten)] - meta: MetadataV1, - }, - V2 { - #[serde(flatten)] - meta: MetadataV2, - }, + V1(MetadataV1), + V2(MetadataV2), } impl Metadata { pub fn new_v2(index_db_size: u64, update_db_size: u64) -> Self { let meta = MetadataV2::new(index_db_size, update_db_size); - Self::V2 { meta } + Self::V2(meta) } } @@ -135,16 +130,31 @@ pub fn load_dump( let mut meta_file = File::open(&meta_path)?; let meta: Metadata = serde_json::from_reader(&mut meta_file)?; + let dst_dir = dst_path + .as_ref() + .parent() + .with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?; + + let tmp_dst = tempfile::tempdir_in(dst_dir)?; + match meta { - Metadata::V1 { meta } => meta.load_dump(&tmp_src_path, dst_path)?, - Metadata::V2 { meta } => meta.load_dump( + Metadata::V1(meta) => meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size as usize)?, + Metadata::V2(meta) => meta.load_dump( &tmp_src_path, - dst_path.as_ref(), + tmp_dst.path(), index_db_size, update_db_size, indexer_opts, )?, } + // Persist and atomically rename the db + let persisted_dump = tmp_dst.into_path(); + if dst_path.as_ref().exists() { + warn!("Overwriting database at {}", dst_path.as_ref().display()); + std::fs::remove_dir_all(&dst_path)?; + } + + std::fs::rename(&persisted_dump, &dst_path)?; Ok(()) } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 2fd9ff301..e666a536e 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -1,13 +1,16 @@ -use std::{collections::HashSet, io::{BufReader, BufRead, Write}}; use std::fs::{create_dir_all, File}; use std::path::{Path, PathBuf}; +use std::{ + collections::HashSet, + io::{BufRead, BufReader, Write}, +}; use heed::{ types::{ByteSlice, Str}, CompactionOption, Database, Env, EnvOpenOptions, }; +use serde::{Deserialize, Serialize}; use uuid::Uuid; -use serde::{Serialize, Deserialize}; use super::{Result, UuidResolverError, UUID_STORE_SIZE}; use crate::helpers::EnvSizer; @@ -45,7 +48,14 @@ impl HeedUuidStore { let mut options = EnvOpenOptions::new(); options.map_size(UUID_STORE_SIZE); // 1GB let env = options.open(path)?; - let db = env.create_database(None)?; Ok(Self { env, db }) } pub fn create_uuid(&self, name: String, err: bool) -> Result { let env = self.env.clone(); let db = self.db; let mut txn = env.write_txn()?; + let db = env.create_database(None)?; + Ok(Self { env, db }) + } + + pub fn create_uuid(&self, name: String, err: bool) -> Result { + let env = self.env.clone(); + let db = self.db; + let mut txn = env.write_txn()?; match db.get(&txn, &name)? { Some(uuid) => { if err { @@ -62,7 +72,10 @@ impl HeedUuidStore { Ok(uuid) } } - } pub fn get_uuid(&self, name: String) -> Result> { let env = self.env.clone(); let db = self.db; + } + pub fn get_uuid(&self, name: String) -> Result> { + let env = self.env.clone(); + let db = self.db; let txn = env.read_txn()?; match db.get(&txn, &name)? { Some(uuid) => { @@ -149,9 +162,7 @@ impl HeedUuidStore { let uid = uid.to_string(); let uuid = Uuid::from_slice(uuid)?; - let entry = DumpEntry { - uuid, uid - }; + let entry = DumpEntry { uuid, uid }; serde_json::to_writer(&mut dump_file, &entry)?; dump_file.write(b"\n").unwrap(); From b3c8f0e1f6156e38a9e74c509cbbfc22f5a0d164 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 31 May 2021 10:58:51 +0200 Subject: [PATCH 358/522] fix empty index error --- meilisearch-http/src/index/dump.rs | 40 ++++++++++++------- .../index_controller/dump_actor/loaders/v1.rs | 28 ++++++------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 9dbb14fbd..eb1d27a4e 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -1,15 +1,15 @@ -use std::{fs::{create_dir_all, File}, path::Path, sync::Arc}; +use std::{fs::{create_dir_all, File}, io::{BufRead, BufReader}, path::Path, sync::Arc}; +use anyhow::bail; use anyhow::Context; use heed::RoTxn; use indexmap::IndexMap; use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; use serde::{Deserialize, Serialize}; -use anyhow::bail; use crate::option::IndexerOpts; -use super::{Unchecked, Index, Settings, update_handler::UpdateHandler}; +use super::{update_handler::UpdateHandler, Index, Settings, Unchecked}; #[derive(Serialize, Deserialize)] struct DumpMeta { @@ -64,7 +64,10 @@ impl Index { let settings = self.settings_txn(txn)?.into_unchecked(); let primary_key = self.primary_key(txn)?.map(String::from); - let meta = DumpMeta { settings, primary_key }; + let meta = DumpMeta { + settings, + primary_key, + }; serde_json::to_writer(&mut meta_file, &meta)?; @@ -86,7 +89,10 @@ impl Index { let meta_path = src.as_ref().join(META_FILE_NAME); let mut meta_file = File::open(meta_path)?; - let DumpMeta { settings, primary_key } = serde_json::from_reader(&mut meta_file)?; + let DumpMeta { + settings, + primary_key, + } = serde_json::from_reader(&mut meta_file)?; let settings = settings.check(); let index = Self::open(&dst_dir_path, size as usize)?; let mut txn = index.write_txn()?; @@ -96,15 +102,21 @@ impl Index { index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?; let document_file_path = src.as_ref().join(DATA_FILE_NAME); - let document_file = File::open(&document_file_path)?; - index.update_documents_txn( - &mut txn, - JsonStream, - IndexDocumentsMethod::UpdateDocuments, - Some(document_file), - handler.update_builder(0), - primary_key.as_deref(), - )?; + let reader = File::open(&document_file_path)?; + let mut reader = BufReader::new(reader); + reader.fill_buf()?; + // If the document file is empty, we don't perform the document addition, to prevent + // a primary key error to be thrown. + if !reader.buffer().is_empty() { + index.update_documents_txn( + &mut txn, + JsonStream, + IndexDocumentsMethod::UpdateDocuments, + Some(reader), + handler.update_builder(0), + primary_key.as_deref(), + )?; + } txn.commit()?; diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 471e66d17..ed268f1f7 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -1,10 +1,4 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - fs::File, - marker::PhantomData, - path::Path, - sync::Arc, -}; +use std::{collections::{BTreeMap, BTreeSet}, fs::File, io::BufRead, marker::PhantomData, path::Path, sync::Arc}; use heed::EnvOpenOptions; use log::{error, info, warn}; @@ -103,15 +97,17 @@ fn load_index( let update_builder = UpdateBuilder::new(0); let file = File::open(&src.as_ref().join("documents.jsonl"))?; - let reader = std::io::BufReader::new(file); - - index.update_documents( - UpdateFormat::JsonStream, - IndexDocumentsMethod::ReplaceDocuments, - Some(reader), - update_builder, - primary_key, - )?; + let mut reader = std::io::BufReader::new(file); + reader.fill_buf()?; + if !reader.buffer().is_empty() { + index.update_documents( + UpdateFormat::JsonStream, + IndexDocumentsMethod::ReplaceDocuments, + Some(reader), + update_builder, + primary_key, + )?; + } // the last step: we extract the original milli::Index and close it Arc::try_unwrap(index.0) From dffbaca63b19ca1b7fb853970b0356a7a1dbf938 Mon Sep 17 00:00:00 2001 From: tamo Date: Mon, 31 May 2021 13:59:31 +0200 Subject: [PATCH 359/522] bump sentry version --- Cargo.lock | 634 ++++++++--------------------------- meilisearch-http/Cargo.toml | 16 +- meilisearch-http/src/main.rs | 36 +- 3 files changed, 167 insertions(+), 519 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1c109a79..6a5cb162c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" version = "0.4.0" @@ -11,9 +13,9 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.6", - "tokio 1.5.0", - "tokio-util 0.6.6", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] @@ -42,7 +44,7 @@ dependencies = [ "actix-tls", "actix-utils", "ahash 0.7.2", - "base64 0.13.0", + "base64", "bitflags", "brotli2", "bytes 1.0.1", @@ -52,7 +54,7 @@ dependencies = [ "flate2", "futures-core", "futures-util", - "h2 0.3.3", + "h2", "http", "httparse", "itoa", @@ -64,14 +66,14 @@ dependencies = [ "paste", "percent-encoding", "pin-project", - "pin-project-lite 0.2.6", + "pin-project-lite", "rand 0.8.3", "regex", "serde", "sha-1 0.9.4", "smallvec", "time 0.2.26", - "tokio 1.5.0", + "tokio", ] [[package]] @@ -105,7 +107,7 @@ checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" dependencies = [ "actix-macros", "futures-core", - "tokio 1.5.0", + "tokio", ] [[package]] @@ -119,10 +121,10 @@ dependencies = [ "actix-utils", "futures-core", "log", - "mio 0.7.11", + "mio", "num_cpus", "slab", - "tokio 1.5.0", + "tokio", ] [[package]] @@ -133,7 +135,7 @@ checksum = "77f5f9d66a8730d0fae62c26f3424f5751e5518086628a40b7ab6fca4a705034" dependencies = [ "futures-core", "paste", - "pin-project-lite 0.2.6", + "pin-project-lite", ] [[package]] @@ -150,9 +152,9 @@ dependencies = [ "futures-core", "http", "log", - "tokio-rustls 0.22.0", - "tokio-util 0.6.6", - "webpki-roots 0.21.1", + "tokio-rustls", + "tokio-util", + "webpki-roots", ] [[package]] @@ -162,7 +164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" dependencies = [ "local-waker", - "pin-project-lite 0.2.6", + "pin-project-lite", ] [[package]] @@ -200,7 +202,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.4.0", + "socket2", "time 0.2.26", "url", ] @@ -277,7 +279,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -341,7 +343,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -371,12 +373,6 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - [[package]] name = "base64" version = "0.13.0" @@ -398,15 +394,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "block-buffer" version = "0.7.3" @@ -502,12 +489,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - [[package]] name = "bytes" version = "0.6.0" @@ -621,7 +602,7 @@ dependencies = [ "num-traits", "serde", "time 0.1.44", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -848,19 +829,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.8.3" @@ -868,34 +836,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" dependencies = [ "atty", - "humantime 2.1.0", + "humantime", "log", "regex", "termcolor", ] -[[package]] -name = "failure" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" -dependencies = [ - "backtrace", - "failure_derive", -] - -[[package]] -name = "failure_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" -dependencies = [ - "proc-macro2 1.0.26", - "quote 1.0.9", - "syn 1.0.72", - "synstructure", -] - [[package]] name = "fake-simd" version = "0.1.2" @@ -911,7 +857,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -975,22 +921,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "futures" version = "0.3.14" @@ -1076,7 +1006,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.6", + "pin-project-lite", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -1159,26 +1089,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "h2" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" -dependencies = [ - "bytes 0.5.6", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio 0.2.25", - "tokio-util 0.3.1", - "tracing", - "tracing-futures", -] - [[package]] name = "h2" version = "0.3.3" @@ -1193,8 +1103,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.5.0", - "tokio-util 0.6.6", + "tokio", + "tokio-util", "tracing", ] @@ -1290,7 +1200,7 @@ checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" dependencies = [ "libc", "match_cfg", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1304,16 +1214,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" -dependencies = [ - "bytes 0.5.6", - "http", -] - [[package]] name = "http-body" version = "0.4.1" @@ -1322,7 +1222,7 @@ checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737" dependencies = [ "bytes 1.0.1", "http", - "pin-project-lite 0.2.6", + "pin-project-lite", ] [[package]] @@ -1349,45 +1249,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" -[[package]] -name = "hyper" -version = "0.13.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" -dependencies = [ - "bytes 0.5.6", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.2.7", - "http", - "http-body 0.3.1", - "httparse", - "httpdate 0.3.2", - "itoa", - "pin-project", - "socket2 0.3.19", - "tokio 0.2.25", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "0.14.7" @@ -1398,36 +1265,20 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.3", + "h2", "http", - "http-body 0.4.1", + "http-body", "httparse", "httpdate 1.0.0", "itoa", "pin-project", - "socket2 0.4.0", - "tokio 1.5.0", + "socket2", + "tokio", "tower-service", "tracing", "want", ] -[[package]] -name = "hyper-rustls" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" -dependencies = [ - "bytes 0.5.6", - "futures-util", - "hyper 0.13.10", - "log", - "rustls 0.18.1", - "tokio 0.2.25", - "tokio-rustls 0.14.1", - "webpki", -] - [[package]] name = "hyper-rustls" version = "0.22.1" @@ -1435,11 +1286,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" dependencies = [ "futures-util", - "hyper 0.14.7", + "hyper", "log", - "rustls 0.19.1", - "tokio 1.5.0", - "tokio-rustls 0.22.0", + "rustls", + "tokio", + "tokio-rustls", "webpki", ] @@ -1454,20 +1305,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "im" -version = "14.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "696059c87b83c5a258817ecd67c3af915e3ed141891fc35a1e79908801cf0ce7" -dependencies = [ - "bitmaps", - "rand_core 0.5.1", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "indexmap" version = "1.6.2" @@ -1488,15 +1325,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "ipnet" version = "2.3.0" @@ -1581,16 +1409,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "language-tags" version = "0.2.2" @@ -1745,7 +1563,7 @@ dependencies = [ "chrono", "crossbeam-channel", "either", - "env_logger 0.8.3", + "env_logger", "flate2", "fst", "futures", @@ -1774,8 +1592,8 @@ dependencies = [ "rand 0.7.3", "rayon", "regex", - "reqwest 0.11.3", - "rustls 0.19.1", + "reqwest", + "rustls", "sentry", "serde", "serde_json", @@ -1789,7 +1607,7 @@ dependencies = [ "tempdir", "tempfile", "thiserror", - "tokio 1.5.0", + "tokio", "urlencoding", "uuid", "vergen", @@ -1826,7 +1644,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1907,25 +1725,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow 0.2.2", - "net2", - "slab", - "winapi 0.2.8", -] - [[package]] name = "mio" version = "0.7.11" @@ -1934,21 +1733,9 @@ checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" dependencies = [ "libc", "log", - "miow 0.3.7", + "miow", "ntapi", - "winapi 0.3.9", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", + "winapi", ] [[package]] @@ -1957,7 +1744,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1987,17 +1774,6 @@ dependencies = [ "syn 1.0.72", ] -[[package]] -name = "net2" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - [[package]] name = "nix" version = "0.19.1" @@ -2022,7 +1798,7 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2106,7 +1882,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2131,7 +1907,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2270,12 +2046,6 @@ dependencies = [ "syn 1.0.72", ] -[[package]] -name = "pin-project-lite" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" - [[package]] name = "pin-project-lite" version = "0.2.6" @@ -2383,12 +2153,6 @@ dependencies = [ "unicode-xid 0.2.2", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" version = "0.6.13" @@ -2417,7 +2181,7 @@ dependencies = [ "libc", "rand_core 0.3.1", "rdrand", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2526,15 +2290,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_xoshiro" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rayon" version = "1.5.0" @@ -2610,44 +2365,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "reqwest" -version = "0.10.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" -dependencies = [ - "base64 0.13.0", - "bytes 0.5.6", - "encoding_rs", - "futures-core", - "futures-util", - "http", - "http-body 0.3.1", - "hyper 0.13.10", - "hyper-rustls 0.21.0", - "ipnet", - "js-sys", - "lazy_static", - "log", - "mime", - "mime_guess", - "percent-encoding", - "pin-project-lite 0.2.6", - "rustls 0.18.1", - "serde", - "serde_json", - "serde_urlencoded", - "tokio 0.2.25", - "tokio-rustls 0.14.1", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots 0.20.0", - "winreg", + "winapi", ] [[package]] @@ -2656,32 +2374,33 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" dependencies = [ - "base64 0.13.0", + "base64", "bytes 1.0.1", "encoding_rs", "futures-core", "futures-util", "http", - "http-body 0.4.1", - "hyper 0.14.7", - "hyper-rustls 0.22.1", + "http-body", + "hyper", + "hyper-rustls", "ipnet", "js-sys", "lazy_static", "log", "mime", "percent-encoding", - "pin-project-lite 0.2.6", - "rustls 0.19.1", + "pin-project-lite", + "rustls", "serde", + "serde_json", "serde_urlencoded", - "tokio 1.5.0", - "tokio-rustls 0.22.0", + "tokio", + "tokio-rustls", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.21.1", + "webpki-roots", "winreg", ] @@ -2703,7 +2422,7 @@ dependencies = [ "spin", "untrusted", "web-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2741,26 +2460,13 @@ dependencies = [ "semver 0.11.0", ] -[[package]] -name = "rustls" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" -dependencies = [ - "base64 0.12.3", - "log", - "ring", - "sct", - "webpki", -] - [[package]] name = "rustls" version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ - "base64 0.13.0", + "base64", "log", "ring", "sct", @@ -2833,39 +2539,90 @@ dependencies = [ [[package]] name = "sentry" -version = "0.18.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b01b723fc1b0a0f9394ca1a8451daec6e20206d47f96c3dceea7fd11ec9eec0" +checksum = "f27c425b07c7186018e2ef9ac3a25b01dae78b05a7ef604d07f216b9f59b42b4" +dependencies = [ + "httpdate 0.3.2", + "reqwest", + "sentry-backtrace", + "sentry-contexts", + "sentry-core", + "sentry-log", + "sentry-panic", +] + +[[package]] +name = "sentry-backtrace" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80a5b9d9be0a0e25b2aaa5f3e9815d7fc6b8904f800c41800e5583652b5ca733" dependencies = [ "backtrace", - "env_logger 0.7.1", - "failure", + "lazy_static", + "regex", + "sentry-core", +] + +[[package]] +name = "sentry-contexts" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2410b212de9b2eb7427d2bf9a1f4f5cb2aa14359863d982066ead00d6de9bce0" +dependencies = [ "hostname", - "httpdate 0.3.2", - "im", "lazy_static", "libc", - "log", - "rand 0.7.3", "regex", - "reqwest 0.10.10", - "rustc_version 0.2.3", - "sentry-types", + "rustc_version 0.3.3", + "sentry-core", "uname", - "url", +] + +[[package]] +name = "sentry-core" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbbe485e384cb5540940e65d729820ffcbedc0c902fcb27081e44dacfe6a0c34" +dependencies = [ + "lazy_static", + "rand 0.8.3", + "sentry-types", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-log" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "647143f672410ae5f242acd40f9f8f39729aff5ac7e856d91450fdfc30c2e960" +dependencies = [ + "log", + "sentry-core", +] + +[[package]] +name = "sentry-panic" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89cf195cff04a50b90e6b9ac8b4874dc63b8e0a466f193702801ef98baa9bd90" +dependencies = [ + "sentry-backtrace", + "sentry-core", ] [[package]] name = "sentry-types" -version = "0.14.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ec406c11c060c8a7d5d67fc6f4beb2888338dcb12b9af409451995f124749d" +checksum = "bc5e777cff85b44538ac766a9604676b7180d01d2566e76b2ac41426c734498c" dependencies = [ "chrono", "debugid", - "failure", "serde", "serde_json", + "thiserror", "url", "uuid", ] @@ -2983,16 +2740,6 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27" -[[package]] -name = "sized-chunks" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59044ea371ad781ff976f7b06480b9f0180e834eda94114f2afb4afc12b7718" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "slab" version = "0.4.3" @@ -3027,17 +2774,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" -[[package]] -name = "socket2" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "winapi 0.3.9", -] - [[package]] name = "socket2" version = "0.4.0" @@ -3045,7 +2781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3217,7 +2953,7 @@ dependencies = [ "rand 0.8.3", "redox_syscall", "remove_dir_all", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3266,7 +3002,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3281,7 +3017,7 @@ dependencies = [ "stdweb", "time-macros", "version_check", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3332,24 +3068,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" -[[package]] -name = "tokio" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" -dependencies = [ - "bytes 0.5.6", - "fnv", - "futures-core", - "iovec", - "lazy_static", - "memchr", - "mio 0.6.23", - "num_cpus", - "pin-project-lite 0.1.12", - "slab", -] - [[package]] name = "tokio" version = "1.5.0" @@ -3360,14 +3078,14 @@ dependencies = [ "bytes 1.0.1", "libc", "memchr", - "mio 0.7.11", + "mio", "num_cpus", "once_cell", "parking_lot", - "pin-project-lite 0.2.6", + "pin-project-lite", "signal-hook-registry", "tokio-macros", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3381,43 +3099,17 @@ dependencies = [ "syn 1.0.72", ] -[[package]] -name = "tokio-rustls" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" -dependencies = [ - "futures-core", - "rustls 0.18.1", - "tokio 0.2.25", - "webpki", -] - [[package]] name = "tokio-rustls" version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ - "rustls 0.19.1", - "tokio 1.5.0", + "rustls", + "tokio", "webpki", ] -[[package]] -name = "tokio-util" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" -dependencies = [ - "bytes 0.5.6", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.1.12", - "tokio 0.2.25", -] - [[package]] name = "tokio-util" version = "0.6.6" @@ -3428,8 +3120,8 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.6", - "tokio 1.5.0", + "pin-project-lite", + "tokio", ] [[package]] @@ -3454,8 +3146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", - "log", - "pin-project-lite 0.2.6", + "pin-project-lite", "tracing-core", ] @@ -3468,16 +3159,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - [[package]] name = "treeline" version = "0.1.0" @@ -3633,7 +3314,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", - "winapi 0.3.9", + "winapi", "winapi-util", ] @@ -3747,15 +3428,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "webpki-roots" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" -dependencies = [ - "webpki", -] - [[package]] name = "webpki-roots" version = "0.21.1" @@ -3774,12 +3446,6 @@ dependencies = [ "hashbrown 0.7.2", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -3790,12 +3456,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -3808,7 +3468,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3823,17 +3483,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] [[package]] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 7ac3ecb38..87b4ddbe2 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -78,17 +78,15 @@ pin-project = "1.0.7" [dependencies.sentry] default-features = false features = [ - "with_client_implementation", - "with_panic", - "with_failure", - "with_device_info", - "with_rust_info", - "with_reqwest_transport", - "with_rustls", - "with_env_logger" + "backtrace", + "contexts", + "panic", + "reqwest", + "rustls", + "log", ] optional = true -version = "0.18.1" +version = "0.22.0" [dev-dependencies] diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index b16f3c0e1..a29299673 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -15,18 +15,8 @@ static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; async fn main() -> Result<(), MainError> { let opt = Opt::from_args(); - #[cfg(all(not(debug_assertions), feature = "sentry"))] - let _sentry = sentry::init(( - if !opt.no_sentry { - Some(opt.sentry_dsn.clone()) - } else { - None - }, - sentry::ClientOptions { - release: sentry::release_name!(), - ..Default::default() - }, - )); + let mut log_builder = + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")); match opt.env.as_ref() { "production" => { @@ -36,16 +26,26 @@ async fn main() -> Result<(), MainError> { .into(), ); } - #[cfg(all(not(debug_assertions), feature = "sentry"))] - if !opt.no_sentry && _sentry.is_enabled() { - sentry::integrations::panic::register_panic_handler(); // TODO: This shouldn't be needed when upgrading to sentry 0.19.0. These integrations are turned on by default when using `sentry::init`. - sentry::integrations::env_logger::init(None, Default::default()); + if !opt.no_sentry { + let logger = sentry::integrations::log::SentryLogger::with_dest(log_builder.build()); + log::set_boxed_logger(Box::new(logger)) + .map(|()| log::set_max_level(log::LevelFilter::Info)) + .unwrap(); + + let sentry = sentry::init(sentry::ClientOptions { + release: sentry::release_name!(), + dsn: Some(opt.sentry_dsn.parse()?), + ..Default::default() + }); + // sentry must stay alive as long as possible + std::mem::forget(sentry); + } else { + log_builder.init(); } } "development" => { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) - .init(); + log_builder.init(); } _ => unreachable!(), } From 10fc870684faa6e7a6014391b450898012f9854b Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 31 May 2021 15:34:03 +0200 Subject: [PATCH 360/522] improve dump info reports --- .../src/index_controller/dump_actor/actor.rs | 68 +++++++++---------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index ff4c39f6d..5ac5ca9b9 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -1,9 +1,9 @@ -use std::path::{Path, PathBuf}; +use std::{collections::HashMap, path::{Path, PathBuf}}; use std::sync::Arc; use async_stream::stream; use chrono::Utc; -use futures::stream::StreamExt; +use futures::{lock::Mutex, stream::StreamExt}; use log::{error, info}; use update_actor::UpdateActorHandle; use uuid_resolver::UuidResolverHandle; @@ -19,7 +19,8 @@ pub struct DumpActor { uuid_resolver: UuidResolver, update: Update, dump_path: PathBuf, - dump_info: Arc>>, + lock: Arc>, + dump_infos: Arc>>, update_db_size: u64, index_db_size: u64, } @@ -42,12 +43,15 @@ where index_db_size: u64, update_db_size: u64, ) -> Self { + let dump_infos = Arc::new(RwLock::new(HashMap::new())); + let lock = Arc::new(Mutex::new(())); Self { inbox: Some(inbox), uuid_resolver, update, dump_path: dump_path.as_ref().into(), - dump_info: Arc::new(RwLock::new(None)), + dump_infos, + lock, index_db_size, update_db_size, } @@ -91,21 +95,22 @@ where } async fn handle_create_dump(&self, ret: oneshot::Sender>) { - if self.is_running().await { - ret.send(Err(DumpError::DumpAlreadyRunning)) - .expect("Dump actor is dead"); - return; - } let uid = generate_uid(); - let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress); - *self.dump_info.write().await = Some(info.clone()); + let _lock = match self.lock.try_lock() { + Some(lock) => lock, + None => { + ret.send(Err(DumpError::DumpAlreadyRunning)) + .expect("Dump actor is dead"); + return; + } + }; + + self.dump_infos.write().await.insert(uid.clone(), info.clone()); ret.send(Ok(info)).expect("Dump actor is dead"); - let dump_info = self.dump_info.clone(); - let task = DumpTask { path: self.dump_path.clone(), uuid_resolver: self.uuid_resolver.clone(), @@ -117,45 +122,34 @@ where let task_result = tokio::task::spawn(task.run()).await; + let mut dump_infos = self.dump_infos + .write() + .await; + let dump_infos = + dump_infos + .get_mut(&uid) + .expect("dump entry deleted while lock was acquired"); + match task_result { Ok(Ok(())) => { - (*dump_info.write().await).as_mut().expect("Inconsistent dump service state").done(); + dump_infos.done(); info!("Dump succeed"); } Ok(Err(e)) => { - (*dump_info.write().await).as_mut().expect("Inconsistent dump service state").with_error(e.to_string()); + dump_infos.with_error(e.to_string()); error!("Dump failed: {}", e); } Err(_) => { + dump_infos.with_error("Unexpected error while performing dump.".to_string()); error!("Dump panicked. Dump status set to failed"); - (*dump_info.write().await).as_mut().expect("Inconsistent dump service state").with_error("Unexpected error while performing dump.".to_string()); } }; } async fn handle_dump_info(&self, uid: String) -> DumpResult { - match &*self.dump_info.read().await { - None => self.dump_from_fs(uid).await, - Some(DumpInfo { uid: ref s, .. }) if &uid != s => self.dump_from_fs(uid).await, + match self.dump_infos.read().await.get(&uid) { Some(info) => Ok(info.clone()), + _ => Err(DumpError::DumpDoesNotExist(uid)), } } - - async fn dump_from_fs(&self, uid: String) -> DumpResult { - self.dump_path - .join(format!("{}.dump", &uid)) - .exists() - .then(|| DumpInfo::new(uid.clone(), DumpStatus::Done)) - .ok_or(DumpError::DumpDoesNotExist(uid)) - } - - async fn is_running(&self) -> bool { - matches!( - *self.dump_info.read().await, - Some(DumpInfo { - status: DumpStatus::InProgress, - .. - }) - ) - } } From 1c4f0b2ccff258fe797414c0a087d4e120024361 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 31 May 2021 16:03:39 +0200 Subject: [PATCH 361/522] clippy, fmt & tests --- meilisearch-http/build.rs | 8 ++- meilisearch-http/src/data/mod.rs | 10 +++- meilisearch-http/src/error.rs | 4 +- .../src/helpers/authentication.rs | 33 +++++------- meilisearch-http/src/index/dump.rs | 11 ++-- meilisearch-http/src/index/mod.rs | 12 +++-- meilisearch-http/src/index/search.rs | 51 +++++++++---------- meilisearch-http/src/index/updates.rs | 2 +- .../src/index_controller/dump_actor/actor.rs | 19 ++++--- .../dump_actor/handle_impl.rs | 15 ++++-- .../index_controller/dump_actor/loaders/v1.rs | 9 +++- .../index_controller/dump_actor/loaders/v2.rs | 6 ++- .../index_controller/dump_actor/message.rs | 4 +- .../src/index_controller/dump_actor/mod.rs | 8 +-- .../src/index_controller/index_actor/actor.rs | 9 ++-- .../index_actor/handle_impl.rs | 5 +- .../index_controller/index_actor/message.rs | 2 +- .../src/index_controller/index_actor/mod.rs | 21 +++++--- .../src/index_controller/snapshot.rs | 6 +-- .../update_actor/store/dump.rs | 18 ++++--- .../update_actor/store/mod.rs | 37 +++++++++----- .../index_controller/uuid_resolver/actor.rs | 2 +- .../index_controller/uuid_resolver/message.rs | 2 +- .../index_controller/uuid_resolver/store.rs | 4 +- meilisearch-http/src/lib.rs | 10 ++-- meilisearch-http/src/routes/dump.rs | 11 ++-- meilisearch-http/src/routes/index.rs | 2 +- meilisearch-http/src/routes/mod.rs | 2 +- meilisearch-http/src/routes/settings/mod.rs | 2 +- meilisearch-http/tests/common/index.rs | 2 +- meilisearch-http/tests/common/server.rs | 2 +- 31 files changed, 196 insertions(+), 133 deletions(-) diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index 5dbde1477..557e04fe7 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -50,7 +50,7 @@ mod mini_dashboard { sha1_file.read_to_string(&mut sha1)?; if sha1 == meta["sha1"].as_str().unwrap() { // Nothing to do. - return Ok(()) + return Ok(()); } } @@ -62,7 +62,11 @@ mod mini_dashboard { hasher.update(&dashboard_assets_bytes); let sha1 = hex::encode(hasher.finalize()); - assert_eq!(meta["sha1"].as_str().unwrap(), sha1, "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml"); + assert_eq!( + meta["sha1"].as_str().unwrap(), + sha1, + "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml" + ); create_dir_all(&dashboard_dir)?; let cursor = Cursor::new(&dashboard_assets_bytes); diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 008065d74..9f8a688bc 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -4,7 +4,9 @@ use std::sync::Arc; use sha2::Digest; use crate::index::{Checked, Settings}; -use crate::index_controller::{IndexController, IndexStats, Stats, DumpInfo, IndexMetadata, IndexSettings}; +use crate::index_controller::{ + DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats, +}; use crate::option::Opt; pub mod search; @@ -67,7 +69,11 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { index_controller, api_keys, options }; + let inner = DataInner { + index_controller, + api_keys, + options, + }; let inner = Arc::new(inner); Ok(Data { inner }) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 6489716ca..07bd96fb9 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -299,7 +299,7 @@ impl From for Error { JsonPayloadError::Payload(err) => { Error::BadRequest(format!("Problem while decoding the request: {}", err)) } - e => Error::Internal(format!("Unexpected Json error: {}", e)) + e => Error::Internal(format!("Unexpected Json error: {}", e)), } } } @@ -310,7 +310,7 @@ impl From for Error { QueryPayloadError::Deserialize(err) => { Error::BadRequest(format!("Invalid query parameters: {}", err)) } - e => Error::Internal(format!("Unexpected query payload error: {}", e)) + e => Error::Internal(format!("Unexpected query payload error: {}", e)), } } } diff --git a/meilisearch-http/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs index a1a0c431e..54d5488f4 100644 --- a/meilisearch-http/src/helpers/authentication.rs +++ b/meilisearch-http/src/helpers/authentication.rs @@ -1,16 +1,16 @@ use std::pin::Pin; use std::task::{Context, Poll}; +use actix_web::body::Body; use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; use actix_web::web; -use actix_web::body::Body; -use futures::ready; -use futures::future::{ok, Future, Ready}; use actix_web::ResponseError as _; +use futures::future::{ok, Future, Ready}; +use futures::ready; use pin_project::pin_project; -use crate::Data; use crate::error::{Error, ResponseError}; +use crate::Data; #[derive(Clone, Copy)] pub enum Authentication { @@ -59,19 +59,15 @@ where let data = req.app_data::>().unwrap(); if data.api_keys().master.is_none() { - return AuthenticationFuture::Authenticated(self.service.call(req)) + return AuthenticationFuture::Authenticated(self.service.call(req)); } let auth_header = match req.headers().get("X-Meili-API-Key") { Some(auth) => match auth.to_str() { Ok(auth) => auth, - Err(_) => { - return AuthenticationFuture::NoHeader(Some(req)) - } + Err(_) => return AuthenticationFuture::NoHeader(Some(req)), }, - None => { - return AuthenticationFuture::NoHeader(Some(req)) - } + None => return AuthenticationFuture::NoHeader(Some(req)), }; let authenticated = match self.acl { @@ -111,15 +107,13 @@ where { type Output = Result, actix_web::Error>; - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) ->Poll { + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.project(); match this { - AuthProj::Authenticated(fut) => { - match ready!(fut.poll(cx)) { - Ok(resp) => Poll::Ready(Ok(resp)), - Err(e) => Poll::Ready(Err(e)), - } - } + AuthProj::Authenticated(fut) => match ready!(fut.poll(cx)) { + Ok(resp) => Poll::Ready(Ok(resp)), + Err(e) => Poll::Ready(Err(e)), + }, AuthProj::NoHeader(req) => { match req.take() { Some(req) => { @@ -135,7 +129,8 @@ where AuthProj::Refused(req) => { match req.take() { Some(req) => { - let bad_token = req.headers() + let bad_token = req + .headers() .get("X-Meili-API-Key") .map(|h| h.to_str().map(String::from).unwrap_or_default()) .unwrap_or_default(); diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index eb1d27a4e..dd29aa50a 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -1,4 +1,9 @@ -use std::{fs::{create_dir_all, File}, io::{BufRead, BufReader}, path::Path, sync::Arc}; +use std::{ + fs::{create_dir_all, File}, + io::{BufRead, BufReader}, + path::Path, + sync::Arc, +}; use anyhow::bail; use anyhow::Context; @@ -17,8 +22,8 @@ struct DumpMeta { primary_key: Option, } -const META_FILE_NAME: &'static str = "meta.json"; -const DATA_FILE_NAME: &'static str = "documents.jsonl"; +const META_FILE_NAME: &str = "meta.json"; +const DATA_FILE_NAME: &str = "documents.jsonl"; impl Index { pub fn dump(&self, path: impl AsRef) -> anyhow::Result<()> { diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 331db07c4..7d9603e9e 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,6 +1,10 @@ -use std::{collections::{BTreeSet, HashSet}, marker::PhantomData, path::Path}; use std::ops::Deref; use std::sync::Arc; +use std::{ + collections::{BTreeSet, HashSet}, + marker::PhantomData, + path::Path, +}; use anyhow::{bail, Context}; use heed::{EnvOpenOptions, RoTxn}; @@ -9,13 +13,13 @@ use serde_json::{Map, Value}; use crate::helpers::EnvSizer; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; -pub use updates::{Facets, Settings, Checked, Unchecked}; use serde::{de::Deserializer, Deserialize}; +pub use updates::{Checked, Facets, Settings, Unchecked}; -mod search; -mod updates; mod dump; +mod search; pub mod update_handler; +mod updates; pub type Document = Map; diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 0ff6c1bc3..bf559eb91 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -90,7 +90,8 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let displayed_ids = self.displayed_fields_ids(&rtxn)? + let displayed_ids = self + .displayed_fields_ids(&rtxn)? .map(|fields| fields.into_iter().collect::>()) .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect()); @@ -156,10 +157,8 @@ impl Index { }; let stop_words = fst::Set::default(); - let highlighter = Highlighter::new( - &stop_words, - (String::from(""), String::from("")), - ); + let highlighter = + Highlighter::new(&stop_words, (String::from(""), String::from(""))); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let document = make_document(&all_attributes, &fields_ids_map, obkv)?; @@ -384,17 +383,16 @@ mod test { #[test] fn no_formatted() { let stop_words = fst::Set::default(); - let highlighter = Highlighter::new( - &stop_words, - (String::from(""), String::from("")), - ); + let highlighter = + Highlighter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let id = fields.insert("test").unwrap(); let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); + obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); @@ -410,8 +408,9 @@ mod test { &highlighter, &matching_words, &all_formatted, - &to_highlight_ids - ).unwrap(); + &to_highlight_ids, + ) + .unwrap(); assert!(value.is_empty()); } @@ -419,17 +418,16 @@ mod test { #[test] fn formatted_no_highlight() { let stop_words = fst::Set::default(); - let highlighter = Highlighter::new( - &stop_words, - (String::from(""), String::from("")), - ); + let highlighter = + Highlighter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let id = fields.insert("test").unwrap(); let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); + obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); @@ -445,8 +443,9 @@ mod test { &highlighter, &matching_words, &all_formatted, - &to_highlight_ids - ).unwrap(); + &to_highlight_ids, + ) + .unwrap(); assert_eq!(value["test"], "hello"); } @@ -454,17 +453,16 @@ mod test { #[test] fn formatted_with_highlight() { let stop_words = fst::Set::default(); - let highlighter = Highlighter::new( - &stop_words, - (String::from(""), String::from("")), - ); + let highlighter = + Highlighter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let id = fields.insert("test").unwrap(); let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); + obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); @@ -480,8 +478,9 @@ mod test { &highlighter, &matching_words, &all_formatted, - &to_highlight_ids - ).unwrap(); + &to_highlight_ids, + ) + .unwrap(); assert_eq!(value["test"], "hello"); } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 566356d5f..5ef6d854e 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -198,7 +198,7 @@ impl Index { builder.index_documents_method(method); //let indexing_callback = - //|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); + //|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); let indexing_callback = |_, _| (); diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 5ac5ca9b9..8ea2e1f6d 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -1,13 +1,16 @@ -use std::{collections::HashMap, path::{Path, PathBuf}}; use std::sync::Arc; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; use async_stream::stream; use chrono::Utc; use futures::{lock::Mutex, stream::StreamExt}; use log::{error, info}; +use tokio::sync::{mpsc, oneshot, RwLock}; use update_actor::UpdateActorHandle; use uuid_resolver::UuidResolverHandle; -use tokio::sync::{mpsc, oneshot, RwLock}; use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus, DumpTask}; use crate::index_controller::{update_actor, uuid_resolver}; @@ -107,7 +110,10 @@ where } }; - self.dump_infos.write().await.insert(uid.clone(), info.clone()); + self.dump_infos + .write() + .await + .insert(uid.clone(), info.clone()); ret.send(Ok(info)).expect("Dump actor is dead"); @@ -122,11 +128,8 @@ where let task_result = tokio::task::spawn(task.run()).await; - let mut dump_infos = self.dump_infos - .write() - .await; - let dump_infos = - dump_infos + let mut dump_infos = self.dump_infos.write().await; + let dump_infos = dump_infos .get_mut(&uid) .expect("dump entry deleted while lock was acquired"); diff --git a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs index ff663798f..3d8665e62 100644 --- a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs @@ -1,7 +1,7 @@ -use std::path::Path; -use actix_web::web::Bytes; -use tokio::sync::{mpsc, oneshot}; use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg, DumpResult}; +use actix_web::web::Bytes; +use std::path::Path; +use tokio::sync::{mpsc, oneshot}; #[derive(Clone)] pub struct DumpActorHandleImpl { @@ -34,7 +34,14 @@ impl DumpActorHandleImpl { update_db_size: u64, ) -> anyhow::Result { let (sender, receiver) = mpsc::channel(10); - let actor = DumpActor::new(receiver, uuid_resolver, update, path, index_db_size, update_db_size); + let actor = DumpActor::new( + receiver, + uuid_resolver, + update, + path, + index_db_size, + update_db_size, + ); tokio::task::spawn(actor.run()); diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index ed268f1f7..70c89664b 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -1,4 +1,11 @@ -use std::{collections::{BTreeMap, BTreeSet}, fs::File, io::BufRead, marker::PhantomData, path::Path, sync::Arc}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs::File, + io::BufRead, + marker::PhantomData, + path::Path, + sync::Arc, +}; use heed::EnvOpenOptions; use log::{error, info, warn}; diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index c0fe0abe6..96001902d 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -4,7 +4,11 @@ use chrono::{DateTime, Utc}; use log::info; use serde::{Deserialize, Serialize}; -use crate::{index::Index, index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}, option::IndexerOpts}; +use crate::{ + index::Index, + index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}, + option::IndexerOpts, +}; #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] diff --git a/meilisearch-http/src/index_controller/dump_actor/message.rs b/meilisearch-http/src/index_controller/dump_actor/message.rs index 14409afbb..dff9f5954 100644 --- a/meilisearch-http/src/index_controller/dump_actor/message.rs +++ b/meilisearch-http/src/index_controller/dump_actor/message.rs @@ -1,7 +1,6 @@ use tokio::sync::oneshot; -use super::{DumpResult, DumpInfo}; - +use super::{DumpInfo, DumpResult}; pub enum DumpMsg { CreateDump { @@ -12,4 +11,3 @@ pub enum DumpMsg { ret: oneshot::Sender>, }, } - diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index e1998f876..dde04bc12 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,13 +1,13 @@ use std::fs::File; use std::path::{Path, PathBuf}; +use anyhow::Context; use chrono::{DateTime, Utc}; use log::{error, info, warn}; #[cfg(test)] use mockall::automock; use serde::{Deserialize, Serialize}; use thiserror::Error; -use anyhow::Context; use loaders::v1::MetadataV1; use loaders::v2::MetadataV2; @@ -25,7 +25,7 @@ mod handle_impl; mod loaders; mod message; -const META_FILE_NAME: &'static str = "metadata.json"; +const META_FILE_NAME: &str = "metadata.json"; pub type DumpResult = std::result::Result; @@ -138,7 +138,9 @@ pub fn load_dump( let tmp_dst = tempfile::tempdir_in(dst_dir)?; match meta { - Metadata::V1(meta) => meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size as usize)?, + Metadata::V1(meta) => { + meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size as usize)? + } Metadata::V2(meta) => meta.load_dump( &tmp_src_path, tmp_dst.path(), diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 2f136c011..31e2a58d4 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -6,14 +6,15 @@ use async_stream::stream; use futures::stream::StreamExt; use heed::CompactionOption; use log::debug; -use tokio::{fs, sync::mpsc}; use tokio::task::spawn_blocking; +use tokio::{fs, sync::mpsc}; use uuid::Uuid; -use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings, update_handler::UpdateHandler}; +use crate::index::{ + update_handler::UpdateHandler, Checked, Document, SearchQuery, SearchResult, Settings, +}; use crate::index_controller::{ - get_arc_ownership_blocking, Failed, IndexStats, Processed, - Processing, + get_arc_ownership_blocking, Failed, IndexStats, Processed, Processing, }; use crate::option::IndexerOpts; diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 26aa189d0..6bf83c647 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -3,7 +3,10 @@ use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use crate::{index::Checked, index_controller::{IndexSettings, IndexStats, Processing}}; +use crate::{ + index::Checked, + index_controller::{IndexSettings, IndexStats, Processing}, +}; use crate::{ index::{Document, SearchQuery, SearchResult, Settings}, index_controller::{Failed, Processed}, diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 714a30ecc..377b2c333 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use tokio::sync::oneshot; use uuid::Uuid; -use crate::index::{Document, SearchQuery, SearchResult, Settings, Checked}; +use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::index_controller::{Failed, IndexStats, Processed, Processing}; use super::{IndexMeta, IndexResult, IndexSettings}; diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index dbea5151d..1ddc0199e 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -15,7 +15,7 @@ use message::IndexMsg; use store::{IndexStore, MapIndexStore}; use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings}; -use crate::index_controller::{Failed, Processed, Processing, IndexStats}; +use crate::index_controller::{Failed, IndexStats, Processed, Processing}; use super::IndexSettings; @@ -44,7 +44,11 @@ impl IndexMeta { let created_at = index.created_at(&txn)?; let updated_at = index.updated_at(&txn)?; let primary_key = index.primary_key(&txn)?.map(String::from); - Ok(Self { created_at, updated_at, primary_key }) + Ok(Self { + created_at, + updated_at, + primary_key, + }) } } @@ -57,7 +61,7 @@ pub enum IndexError { #[error("Existing primary key")] ExistingPrimaryKey, #[error("Internal Index Error: {0}")] - Internal(String) + Internal(String), } macro_rules! internal_error { @@ -72,7 +76,12 @@ macro_rules! internal_error { } } -internal_error!(anyhow::Error, heed::Error, tokio::task::JoinError, std::io::Error); +internal_error!( + anyhow::Error, + heed::Error, + tokio::task::JoinError, + std::io::Error +); #[async_trait::async_trait] #[cfg_attr(test, automock)] @@ -190,8 +199,8 @@ mod test { self.as_ref().snapshot(uuid, path).await } - async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()> { - self.as_ref().dump(uid, uuid, path).await + async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + self.as_ref().dump(uuid, path).await } async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 2a456eb26..daef7d582 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -144,7 +144,7 @@ mod test { use crate::index_controller::update_actor::{ MockUpdateActorHandle, UpdateActorHandleImpl, UpdateError, }; - use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidError}; + use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidResolverError}; #[actix_rt::test] async fn test_normal() { @@ -193,7 +193,7 @@ mod test { .expect_snapshot() .times(1) // abitrary error - .returning(|_| Box::pin(err(UuidError::NameAlreadyExist))); + .returning(|_| Box::pin(err(UuidResolverError::NameAlreadyExist))); let update_handle = MockUpdateActorHandle::new(); @@ -248,7 +248,7 @@ mod test { // we expect the funtion to be called between 2 and 3 time in the given interval. .times(2..4) // abitrary error, to short-circuit the function - .returning(move |_| Box::pin(err(UuidError::NameAlreadyExist))); + .returning(move |_| Box::pin(err(UuidResolverError::NameAlreadyExist))); let update_handle = MockUpdateActorHandle::new(); diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index fad8974f3..6dfb300e2 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -11,7 +11,10 @@ use uuid::Uuid; use super::UpdateStore; use super::{codec::UpdateKeyCodec, State}; -use crate::index_controller::{Enqueued, UpdateStatus, index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path}; +use crate::index_controller::{ + index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued, + UpdateStatus, +}; #[derive(Serialize, Deserialize)] struct UpdateEntry { @@ -89,7 +92,7 @@ impl UpdateStore { }; serde_json::to_writer(&mut file, &update_json)?; - file.write(b"\n")?; + file.write_all(b"\n")?; } } @@ -111,12 +114,12 @@ impl UpdateStore { for update in updates { let ((uuid, _), data) = update?; if uuids.contains(&uuid) { - let update = data.decode()?.into(); + let update = data.decode()?; let update_json = UpdateEntry { uuid, update }; serde_json::to_writer(&mut file, &update_json)?; - file.write(b"\n")?; + file.write_all(b"\n")?; } } @@ -131,7 +134,6 @@ impl UpdateStore { let dst_update_path = dst.as_ref().join("updates/"); create_dir_all(&dst_update_path)?; - let mut options = EnvOpenOptions::new(); options.map_size(db_size as usize); let (store, _) = UpdateStore::new(options, &dst_update_path)?; @@ -152,7 +154,11 @@ impl UpdateStore { store.register_raw_updates(&mut wtxn, &update, uuid)?; // Copy ascociated update path if it exists - if let UpdateStatus::Enqueued(Enqueued { content: Some(uuid), .. }) = update { + if let UpdateStatus::Enqueued(Enqueued { + content: Some(uuid), + .. + }) = update + { let src = update_uuid_to_file_path(&src_update_path, uuid); let dst = update_uuid_to_file_path(&dst_update_path, uuid); std::fs::copy(src, dst)?; diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 29ccd4f34..006549fb6 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -1,10 +1,13 @@ -pub mod dump; mod codec; +pub mod dump; -use std::{collections::{BTreeMap, HashSet}, path::PathBuf}; use std::fs::{copy, create_dir_all, remove_file, File}; use std::path::Path; use std::sync::Arc; +use std::{ + collections::{BTreeMap, HashSet}, + path::PathBuf, +}; use arc_swap::ArcSwap; use futures::StreamExt; @@ -20,13 +23,13 @@ use uuid::Uuid; use codec::*; use super::UpdateMeta; -use crate::{helpers::EnvSizer, index_controller::index_actor::IndexResult}; use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle}; +use crate::{helpers::EnvSizer, index_controller::index_actor::IndexResult}; #[allow(clippy::upper_case_acronyms)] type BEU64 = U64; -const UPDATE_DIR: &'static str = "update_files"; +const UPDATE_DIR: &str = "update_files"; pub struct UpdateStoreInfo { /// Size of the update store in bytes. @@ -441,11 +444,12 @@ impl UpdateStore { txn.commit()?; - uuids_to_remove.iter() + uuids_to_remove + .iter() .map(|uuid| update_uuid_to_file_path(&self.path, *uuid)) .for_each(|path| { - let _ = remove_file(path); - }); + let _ = remove_file(path); + }); // We don't care about the currently processing update, since it will be removed by itself // once its done processing, and we can't abort a running update. @@ -482,7 +486,11 @@ impl UpdateStore { for entry in pendings { let ((_, uuid, _), pending) = entry?; if uuids.contains(&uuid) { - if let Enqueued { content: Some(uuid), .. } = pending.decode()? { + if let Enqueued { + content: Some(uuid), + .. + } = pending.decode()? + { let path = update_uuid_to_file_path(&self.path, uuid); copy(path, &update_files_path)?; } @@ -507,13 +515,16 @@ impl UpdateStore { Ok(()) } - pub fn get_info(&self) -> anyhow::Result { let mut size = self.env.size(); let txn = self.env.read_txn()?; for entry in self.pending_queue.iter(&txn)? { let (_, pending) = entry?; - if let Enqueued { content: Some(uuid), .. } = pending { + if let Enqueued { + content: Some(uuid), + .. + } = pending + { let path = update_uuid_to_file_path(&self.path, uuid); size += File::open(path)?.metadata()?.len(); } @@ -528,7 +539,9 @@ impl UpdateStore { } fn update_uuid_to_file_path(root: impl AsRef, uuid: Uuid) -> PathBuf { - root.as_ref().join(UPDATE_DIR).join(format!("update_{}", uuid)) + root.as_ref() + .join(UPDATE_DIR) + .join(format!("update_{}", uuid)) } #[cfg(test)] @@ -577,7 +590,7 @@ mod test { let store_clone = update_store.clone(); tokio::task::spawn_blocking(move || { store_clone - .register_update(meta, Some("here"), uuid) + .register_update(meta, None, uuid) .unwrap(); }) .await diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 3592c3551..0211cef25 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -4,7 +4,7 @@ use log::{info, warn}; use tokio::sync::mpsc; use uuid::Uuid; -use super::{Result, UuidResolverError, UuidResolveMsg, UuidStore}; +use super::{Result, UuidResolveMsg, UuidResolverError, UuidStore}; pub struct UuidResolverActor { inbox: mpsc::Receiver, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index 166347455..2092c67fd 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -37,5 +37,5 @@ pub enum UuidResolveMsg { DumpRequest { path: PathBuf, ret: oneshot::Sender>>, - } + }, } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index e666a536e..6289cefcd 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -164,7 +164,7 @@ impl HeedUuidStore { let entry = DumpEntry { uuid, uid }; serde_json::to_writer(&mut dump_file, &entry)?; - dump_file.write(b"\n").unwrap(); + dump_file.write_all(b"\n").unwrap(); uuids.insert(uuid); } @@ -192,7 +192,7 @@ impl HeedUuidStore { println!("importing {} {}", uid, uuid); db.db.put(&mut txn, &uid, uuid.as_bytes())?; } - Err(e) => Err(e)?, + Err(e) => return Err(e.into()), } line.clear(); diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index e19037482..26b6a784c 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -62,11 +62,11 @@ macro_rules! create_app { app.wrap( Cors::default() - .send_wildcard() - .allowed_headers(vec!["content-type", "x-meili-api-key"]) - .allow_any_origin() - .allow_any_method() - .max_age(86_400) // 24h + .send_wildcard() + .allowed_headers(vec!["content-type", "x-meili-api-key"]) + .allow_any_origin() + .allow_any_method() + .max_age(86_400), // 24h ) .wrap(middleware::Logger::default()) .wrap(middleware::Compress::default()) diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 47c081e6f..370eef509 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -1,20 +1,17 @@ -use actix_web::{post, get, web}; use actix_web::HttpResponse; -use serde::{Serialize, Deserialize}; +use actix_web::{get, post, web}; +use serde::{Deserialize, Serialize}; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(create_dump) - .service(get_dump_status); + cfg.service(create_dump).service(get_dump_status); } #[post("/dumps", wrap = "Authentication::Private")] -async fn create_dump( - data: web::Data, -) -> Result { +async fn create_dump(data: web::Data) -> Result { let res = data.create_dump().await?; Ok(HttpResponse::Accepted().json(res)) diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 62717c90d..4dfe90abf 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -1,7 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use crate::error::ResponseError; use crate::helpers::Authentication; diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 999c4f881..a550064ba 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -2,6 +2,7 @@ use actix_web::{get, HttpResponse}; use serde::{Deserialize, Serialize}; pub mod document; +pub mod dump; pub mod health; pub mod index; pub mod key; @@ -9,7 +10,6 @@ pub mod search; pub mod settings; pub mod stats; pub mod synonym; -pub mod dump; #[derive(Deserialize)] pub struct IndexParam { diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 03f1ee95c..8ede56046 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -1,9 +1,9 @@ use actix_web::{delete, get, post, web, HttpResponse}; -use crate::{error::ResponseError, index::Unchecked}; use crate::helpers::Authentication; use crate::index::Settings; use crate::Data; +use crate::{error::ResponseError, index::Unchecked}; #[macro_export] macro_rules! make_setting_route { diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index adb7fef3e..7d98d0733 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -47,7 +47,7 @@ impl Index<'_> { update_id as u64 } - pub async fn create(& self, primary_key: Option<&str>) -> (Value, StatusCode) { + pub async fn create(&self, primary_key: Option<&str>) -> (Value, StatusCode) { let body = json!({ "uid": self.uid, "primaryKey": primary_key, diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 100722ec4..3c50110c3 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -44,7 +44,7 @@ impl Server { } /// Returns a view to an index. There is no guarantee that the index exists. - pub fn index(& self, uid: impl AsRef) -> Index<'_> { + pub fn index(&self, uid: impl AsRef) -> Index<'_> { Index { uid: encode(uid.as_ref()), service: &self.service, From 6609f9e3bec9ea833d0a2e70d2ef5f51565bd994 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 31 May 2021 16:40:59 +0200 Subject: [PATCH 362/522] review edits --- meilisearch-http/src/index/dump.rs | 19 +++--- meilisearch-http/src/index/mod.rs | 13 ++-- meilisearch-http/src/index/updates.rs | 6 +- .../src/index_controller/dump_actor/actor.rs | 14 ++-- .../dump_actor/handle_impl.rs | 10 +-- .../index_controller/dump_actor/loaders/v1.rs | 64 +++++++++++-------- .../index_controller/dump_actor/loaders/v2.rs | 25 ++++---- .../src/index_controller/dump_actor/mod.rs | 14 ++-- meilisearch-http/src/index_controller/mod.rs | 8 +-- .../index_controller/update_actor/actor.rs | 2 +- .../update_actor/store/dump.rs | 2 +- .../update_actor/store/mod.rs | 4 +- .../index_controller/uuid_resolver/store.rs | 24 ++++--- 13 files changed, 100 insertions(+), 105 deletions(-) diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index dd29aa50a..247c02085 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -1,12 +1,9 @@ -use std::{ - fs::{create_dir_all, File}, - io::{BufRead, BufReader}, - path::Path, - sync::Arc, -}; +use std::fs::{create_dir_all, File}; +use std::io::{BufRead, BufReader, Write}; +use std::path::Path; +use std::sync::Arc; -use anyhow::bail; -use anyhow::Context; +use anyhow::{bail, Context}; use heed::RoTxn; use indexmap::IndexMap; use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; @@ -55,7 +52,7 @@ impl Index { } serde_json::to_writer(&mut document_file, &json_map)?; - std::io::Write::write(&mut document_file, b"\n")?; + document_file.write_all(b"\n")?; json_map.clear(); } @@ -82,7 +79,7 @@ impl Index { pub fn load_dump( src: impl AsRef, dst: impl AsRef, - size: u64, + size: usize, indexing_options: &IndexerOpts, ) -> anyhow::Result<()> { let dir_name = src @@ -99,7 +96,7 @@ impl Index { primary_key, } = serde_json::from_reader(&mut meta_file)?; let settings = settings.check(); - let index = Self::open(&dst_dir_path, size as usize)?; + let index = Self::open(&dst_dir_path, size)?; let mut txn = index.write_txn()?; let handler = UpdateHandler::new(&indexing_options)?; diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 7d9603e9e..790ac58f0 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,10 +1,9 @@ +use std::collections::{BTreeSet, HashSet}; +use std::fs::create_dir_all; +use std::marker::PhantomData; use std::ops::Deref; +use std::path::Path; use std::sync::Arc; -use std::{ - collections::{BTreeSet, HashSet}, - marker::PhantomData, - path::Path, -}; use anyhow::{bail, Context}; use heed::{EnvOpenOptions, RoTxn}; @@ -44,7 +43,7 @@ where impl Index { pub fn open(path: impl AsRef, size: usize) -> anyhow::Result { - std::fs::create_dir_all(&path)?; + create_dir_all(&path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); let index = milli::Index::new(options, &path)?; @@ -113,8 +112,6 @@ impl Index { let mut documents = Vec::new(); - println!("fields to display: {:?}", fields_to_display); - for entry in iter { let (_id, obkv) = entry?; let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 5ef6d854e..046823fb7 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -197,10 +197,8 @@ impl Index { builder.update_format(format); builder.index_documents_method(method); - //let indexing_callback = - //|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); - - let indexing_callback = |_, _| (); + let indexing_callback = + |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); let gzipped = false; let addition = match content { diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 8ea2e1f6d..c78079de6 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -1,8 +1,6 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; use async_stream::stream; use chrono::Utc; @@ -24,8 +22,8 @@ pub struct DumpActor { dump_path: PathBuf, lock: Arc>, dump_infos: Arc>>, - update_db_size: u64, - index_db_size: u64, + update_db_size: usize, + index_db_size: usize, } /// Generate uid from creation date @@ -43,8 +41,8 @@ where uuid_resolver: UuidResolver, update: Update, dump_path: impl AsRef, - index_db_size: u64, - update_db_size: u64, + index_db_size: usize, + update_db_size: usize, ) -> Self { let dump_infos = Arc::new(RwLock::new(HashMap::new())); let lock = Arc::new(Mutex::new(())); diff --git a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs index 3d8665e62..ab91aeae6 100644 --- a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs @@ -1,8 +1,10 @@ -use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg, DumpResult}; -use actix_web::web::Bytes; use std::path::Path; + +use actix_web::web::Bytes; use tokio::sync::{mpsc, oneshot}; +use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg, DumpResult}; + #[derive(Clone)] pub struct DumpActorHandleImpl { sender: mpsc::Sender, @@ -30,8 +32,8 @@ impl DumpActorHandleImpl { path: impl AsRef, uuid_resolver: crate::index_controller::uuid_resolver::UuidResolverHandleImpl, update: crate::index_controller::update_actor::UpdateActorHandleImpl, - index_db_size: u64, - update_db_size: u64, + index_db_size: usize, + update_db_size: usize, ) -> anyhow::Result { let (sender, receiver) = mpsc::channel(10); let actor = DumpActor::new( diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 70c89664b..89893998e 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -1,22 +1,20 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - fs::File, - io::BufRead, - marker::PhantomData, - path::Path, - sync::Arc, -}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{create_dir_all, File}; +use std::io::BufRead; +use std::marker::PhantomData; +use std::path::Path; +use std::sync::Arc; use heed::EnvOpenOptions; use log::{error, info, warn}; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::{index::deserialize_some, index_controller::uuid_resolver::HeedUuidStore}; +use crate::index_controller::{self, uuid_resolver::HeedUuidStore, IndexMetadata}; use crate::{ - index::{Index, Unchecked}, - index_controller::{self, IndexMetadata}, + index::{deserialize_some, update_handler::UpdateHandler, Index, Unchecked}, + option::IndexerOpts, }; #[derive(Serialize, Deserialize, Debug)] @@ -32,28 +30,33 @@ impl MetadataV1 { src: impl AsRef, dst: impl AsRef, size: usize, + indexer_options: &IndexerOpts, ) -> anyhow::Result<()> { info!( "Loading dump, dump database version: {}, dump version: V1", self.db_version ); - dbg!("here"); - let uuid_store = HeedUuidStore::new(&dst)?; - dbg!("here"); for index in self.indexes { let uuid = Uuid::new_v4(); uuid_store.insert(index.uid.clone(), uuid)?; let src = src.as_ref().join(index.uid); - load_index(&src, &dst, uuid, index.meta.primary_key.as_deref(), size)?; + load_index( + &src, + &dst, + uuid, + index.meta.primary_key.as_deref(), + size, + indexer_options, + )?; } Ok(()) } } -//This is the settings used in the last version of meilisearch exporting dump in V1 +// These are the settings used in legacy meilisearch (, size: usize, + indexer_options: &IndexerOpts, ) -> anyhow::Result<()> { let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid)); - std::fs::create_dir_all(&index_path)?; + create_dir_all(&index_path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); let index = milli::Index::new(options, index_path)?; @@ -99,31 +103,37 @@ fn load_index( // extract `settings.json` file and import content let settings = import_settings(&src)?; let settings: index_controller::Settings = settings.into(); - let update_builder = UpdateBuilder::new(0); - index.update_settings(&settings.check(), update_builder)?; - let update_builder = UpdateBuilder::new(0); + let mut txn = index.write_txn()?; + + let handler = UpdateHandler::new(&indexer_options)?; + + index.update_settings_txn(&mut txn, &settings.check(), handler.update_builder(0))?; + let file = File::open(&src.as_ref().join("documents.jsonl"))?; let mut reader = std::io::BufReader::new(file); reader.fill_buf()?; if !reader.buffer().is_empty() { - index.update_documents( + index.update_documents_txn( + &mut txn, UpdateFormat::JsonStream, IndexDocumentsMethod::ReplaceDocuments, Some(reader), - update_builder, + handler.update_builder(0), primary_key, )?; } - // the last step: we extract the original milli::Index and close it + txn.commit()?; + + // Finaly, we extract the original milli::Index and close it Arc::try_unwrap(index.0) - .map_err(|_e| "[dumps] At this point no one is supposed to have a reference on the index") + .map_err(|_e| "Couln't close index properly") .unwrap() .prepare_for_closing() .wait(); - // Ignore updates in v1. + // Updates are ignored in dumps V1. Ok(()) } @@ -172,7 +182,7 @@ impl From for index_controller::Settings { /// Extract Settings from `settings.json` file present at provided `dir_path` fn import_settings(dir_path: impl AsRef) -> anyhow::Result { - let path = dbg!(dir_path.as_ref().join("settings.json")); + let path = dir_path.as_ref().join("settings.json"); let file = File::open(path)?; let reader = std::io::BufReader::new(file); let metadata = serde_json::from_reader(reader)?; diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index 96001902d..eddd8a3b7 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -4,23 +4,21 @@ use chrono::{DateTime, Utc}; use log::info; use serde::{Deserialize, Serialize}; -use crate::{ - index::Index, - index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}, - option::IndexerOpts, -}; +use crate::index::Index; +use crate::index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}; +use crate::option::IndexerOpts; #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] pub struct MetadataV2 { db_version: String, - index_db_size: u64, - update_db_size: u64, + index_db_size: usize, + update_db_size: usize, dump_date: DateTime, } impl MetadataV2 { - pub fn new(index_db_size: u64, update_db_size: u64) -> Self { + pub fn new(index_db_size: usize, update_db_size: usize) -> Self { Self { db_version: env!("CARGO_PKG_VERSION").to_string(), index_db_size, @@ -33,9 +31,8 @@ impl MetadataV2 { self, src: impl AsRef, dst: impl AsRef, - // TODO: use these variable to test if loading the index is possible. - _index_db_size: u64, - _update_db_size: u64, + index_db_size: usize, + update_db_size: usize, indexing_options: &IndexerOpts, ) -> anyhow::Result<()> { info!( @@ -47,14 +44,14 @@ impl MetadataV2 { HeedUuidStore::load_dump(src.as_ref(), &dst)?; info!("Loading updates."); - UpdateStore::load_dump(&src, &dst, self.update_db_size)?; + UpdateStore::load_dump(&src, &dst, update_db_size)?; - info!("Loading indexes"); + info!("Loading indexes."); let indexes_path = src.as_ref().join("indexes"); let indexes = indexes_path.read_dir()?; for index in indexes { let index = index?; - Index::load_dump(&index.path(), &dst, self.index_db_size, indexing_options)?; + Index::load_dump(&index.path(), &dst, index_db_size, indexing_options)?; } Ok(()) diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index dde04bc12..0bddaf7a3 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -8,6 +8,7 @@ use log::{error, info, warn}; use mockall::automock; use serde::{Deserialize, Serialize}; use thiserror::Error; +use tokio::fs::create_dir_all; use loaders::v1::MetadataV1; use loaders::v2::MetadataV2; @@ -15,7 +16,6 @@ use loaders::v2::MetadataV2; pub use actor::DumpActor; pub use handle_impl::*; pub use message::DumpMsg; -use tokio::fs::create_dir_all; use super::{update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle}; use crate::{helpers::compression, option::IndexerOpts}; @@ -61,7 +61,7 @@ pub enum Metadata { } impl Metadata { - pub fn new_v2(index_db_size: u64, update_db_size: u64) -> Self { + pub fn new_v2(index_db_size: usize, update_db_size: usize) -> Self { let meta = MetadataV2::new(index_db_size, update_db_size); Self::V2(meta) } @@ -117,8 +117,8 @@ impl DumpInfo { pub fn load_dump( dst_path: impl AsRef, src_path: impl AsRef, - index_db_size: u64, - update_db_size: u64, + index_db_size: usize, + update_db_size: usize, indexer_opts: &IndexerOpts, ) -> anyhow::Result<()> { let tmp_src = tempfile::tempdir_in(".")?; @@ -139,7 +139,7 @@ pub fn load_dump( match meta { Metadata::V1(meta) => { - meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size as usize)? + meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size, indexer_opts)? } Metadata::V2(meta) => meta.load_dump( &tmp_src_path, @@ -166,8 +166,8 @@ struct DumpTask { uuid_resolver: U, update_handle: P, uid: String, - update_db_size: u64, - index_db_size: u64, + update_db_size: usize, + index_db_size: usize, } impl DumpTask diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 18ba6dee3..0615bb731 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -97,8 +97,8 @@ impl IndexController { load_dump( &options.db_path, src_path, - options.max_mdb_size.get_bytes(), - options.max_udb_size.get_bytes(), + options.max_mdb_size.get_bytes() as usize, + options.max_udb_size.get_bytes() as usize, &options.indexer_options, )?; } @@ -116,8 +116,8 @@ impl IndexController { &options.dumps_dir, uuid_resolver.clone(), update_handle.clone(), - options.max_mdb_size.get_bytes(), - options.max_udb_size.get_bytes(), + options.max_mdb_size.get_bytes() as usize, + options.max_udb_size.get_bytes() as usize, )?; if options.schedule_snapshot { diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 40bba4e2b..7779f2556 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -197,7 +197,7 @@ where async fn handle_dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); - println!("starting dump"); + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { update_store.dump(&uuids, path.to_path_buf(), index_handle)?; Ok(()) diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index 6dfb300e2..8f947e459 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -129,7 +129,7 @@ impl UpdateStore { pub fn load_dump( src: impl AsRef, dst: impl AsRef, - db_size: u64, + db_size: usize, ) -> anyhow::Result<()> { let dst_update_path = dst.as_ref().join("updates/"); create_dir_all(&dst_update_path)?; diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 006549fb6..28204f4c0 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -589,9 +589,7 @@ mod test { let uuid = Uuid::new_v4(); let store_clone = update_store.clone(); tokio::task::spawn_blocking(move || { - store_clone - .register_update(meta, None, uuid) - .unwrap(); + store_clone.register_update(meta, None, uuid).unwrap(); }) .await .unwrap(); diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 6289cefcd..1d6ada269 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -1,14 +1,10 @@ +use std::collections::HashSet; use std::fs::{create_dir_all, File}; +use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; -use std::{ - collections::HashSet, - io::{BufRead, BufReader, Write}, -}; -use heed::{ - types::{ByteSlice, Str}, - CompactionOption, Database, Env, EnvOpenOptions, -}; +use heed::types::{ByteSlice, Str}; +use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -21,6 +17,8 @@ struct DumpEntry { uid: String, } +const UUIDS_DB_PATH: &str = "index_uuids"; + #[async_trait::async_trait] pub trait UuidStore: Sized { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return @@ -43,7 +41,7 @@ pub struct HeedUuidStore { impl HeedUuidStore { pub fn new(path: impl AsRef) -> anyhow::Result { - let path = path.as_ref().join("index_uuids"); + let path = path.as_ref().join(UUIDS_DB_PATH); create_dir_all(&path)?; let mut options = EnvOpenOptions::new(); options.map_size(UUID_STORE_SIZE); // 1GB @@ -137,7 +135,7 @@ impl HeedUuidStore { // only perform snapshot if there are indexes if !entries.is_empty() { - path.push("index_uuids"); + path.push(UUIDS_DB_PATH); create_dir_all(&path).unwrap(); path.push("data.mdb"); env.copy_to_path(path, CompactionOption::Enabled)?; @@ -150,7 +148,7 @@ impl HeedUuidStore { } pub fn dump(&self, path: PathBuf) -> Result> { - let dump_path = path.join("index_uuids"); + let dump_path = path.join(UUIDS_DB_PATH); create_dir_all(&dump_path)?; let dump_file_path = dump_path.join("data.jsonl"); let mut dump_file = File::create(&dump_file_path)?; @@ -173,10 +171,10 @@ impl HeedUuidStore { } pub fn load_dump(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { - let uuid_resolver_path = dst.as_ref().join("uuid_resolver/"); + let uuid_resolver_path = dst.as_ref().join(UUIDS_DB_PATH); std::fs::create_dir_all(&uuid_resolver_path)?; - let src_indexes = src.as_ref().join("index_uuids/data.jsonl"); + let src_indexes = src.as_ref().join(UUIDS_DB_PATH).join("data.jsonl"); let indexes = File::open(&src_indexes)?; let mut indexes = BufReader::new(indexes); let mut line = String::new(); From df6ba0e8246c23e18ad6aede46035c71de0b742f Mon Sep 17 00:00:00 2001 From: marin Date: Tue, 1 Jun 2021 11:18:37 +0200 Subject: [PATCH 363/522] Apply suggestions from code review Co-authored-by: Irevoire --- meilisearch-http/src/index/dump.rs | 2 +- .../src/index_controller/dump_actor/loaders/v1.rs | 10 +--------- .../src/index_controller/dump_actor/mod.rs | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 247c02085..13e6cbc02 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -24,7 +24,7 @@ const DATA_FILE_NAME: &str = "documents.jsonl"; impl Index { pub fn dump(&self, path: impl AsRef) -> anyhow::Result<()> { - // acquire write txn make sure any ongoing write is finnished before we start. + // acquire write txn make sure any ongoing write is finished before we start. let txn = self.env.write_txn()?; self.dump_documents(&txn, &path)?; diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 89893998e..decd67f87 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -76,14 +76,6 @@ struct Settings { pub attributes_for_faceting: Option>>, } -impl std::ops::Deref for Settings { - type Target = Option>>; - - fn deref(&self) -> &Self::Target { - &self.stop_words - } -} - fn load_index( src: impl AsRef, dst: impl AsRef, @@ -128,7 +120,7 @@ fn load_index( // Finaly, we extract the original milli::Index and close it Arc::try_unwrap(index.0) - .map_err(|_e| "Couln't close index properly") + .map_err(|_e| "Couldn't close the index properly") .unwrap() .prepare_for_closing() .wait(); diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 0bddaf7a3..66f081e87 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -200,7 +200,7 @@ where let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; - let dump_path = self.path.join(format!("{}.dump", self.uid)); + let dump_path = self.path.join(self.uid).with_extension("dump"); temp_dump_file.persist(&dump_path)?; Ok(dump_path) From d0552e765e8fecda614c6ded7b3505d3cc62fbc9 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 1 Jun 2021 20:15:51 +0200 Subject: [PATCH 364/522] forbid deserialization of Setting --- meilisearch-http/src/index/update_handler.rs | 2 +- meilisearch-http/src/index/updates.rs | 5 +++-- meilisearch-http/src/index_controller/mod.rs | 2 +- meilisearch-http/src/index_controller/updates.rs | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs index 6a303b4ce..8a127168e 100644 --- a/meilisearch-http/src/index/update_handler.rs +++ b/meilisearch-http/src/index/update_handler.rs @@ -82,7 +82,7 @@ impl UpdateHandler { ), ClearDocuments => index.clear_documents(update_builder), DeleteDocuments => index.delete_documents(content, update_builder), - Settings(settings) => index.update_settings(settings, update_builder), + Settings(settings) => index.update_settings(&settings.clone().check(), update_builder), }; match result { diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 046823fb7..b4869fa42 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -20,14 +20,15 @@ where s.serialize_some(&field.as_ref().map(|o| o.as_ref().unwrap_or(&wildcard))) } -#[derive(Clone, Default, Debug)] +#[derive(Clone, Default, Debug, Serialize)] pub struct Checked; -#[derive(Clone, Default, Debug)] +#[derive(Clone, Default, Debug, Serialize, Deserialize)] pub struct Unchecked; #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] +#[serde(bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>"))] pub struct Settings { #[serde( default, diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 0615bb731..f562d2185 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -223,7 +223,7 @@ impl IndexController { create: bool, ) -> anyhow::Result { let perform_udpate = |uuid| async move { - let meta = UpdateMeta::Settings(settings); + let meta = UpdateMeta::Settings(settings.into_unchecked()); // Nothing so send, drop the sender right away, as not to block the update actor. let (_, receiver) = mpsc::channel(1); self.update_handle.update(meta, receiver, uuid).await diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 0aacf9b6c..303289df3 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -3,7 +3,7 @@ use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::index::{Checked, Settings}; +use crate::index::{Unchecked, Settings}; pub type UpdateError = String; @@ -24,7 +24,7 @@ pub enum UpdateMeta { }, ClearDocuments, DeleteDocuments, - Settings(Settings), + Settings(Settings), } #[derive(Debug, Serialize, Deserialize, Clone)] From 2d7785ae0c5f8846d89b52d0ccb9e74b30277a60 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 1 Jun 2021 20:27:12 +0200 Subject: [PATCH 365/522] remove the dump_batch_size option from the CLI --- meilisearch-http/src/option.rs | 5 ----- meilisearch-http/tests/common/server.rs | 1 - 2 files changed, 6 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 87238c4d7..eb81ab9fd 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -202,11 +202,6 @@ pub struct Opt { #[structopt(long, conflicts_with = "import-snapshot")] pub import_dump: Option, - /// The batch size used in the importation process, the bigger it is the faster the dump is created. - /// This options is now deprecated and will be ignored - #[structopt(long, env = "MEILI_DUMP_BATCH_SIZE", default_value = "1024")] - pub dump_batch_size: usize, - #[structopt(flatten)] pub indexer_options: IndexerOpts, } diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 3c50110c3..0fb801d7f 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -68,7 +68,6 @@ pub fn default_settings(dir: impl AsRef) -> Opt { Opt { db_path: dir.as_ref().join("db"), dumps_dir: dir.as_ref().join("dump"), - dump_batch_size: 16, http_addr: "127.0.0.1:7700".to_owned(), master_key: None, env: "development".to_owned(), From 75c0718691f1efd4eb004dde1aa9aaec08a7ba2e Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 2 Jun 2021 15:20:32 +0200 Subject: [PATCH 366/522] fix update loop infinite loop --- .../src/index_controller/index_actor/actor.rs | 4 +--- .../src/index_controller/update_actor/store/mod.rs | 8 ++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 31e2a58d4..4d8324e11 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -168,9 +168,7 @@ impl IndexActor { None => self.store.create(uuid, None).await?, }; - let result = - spawn_blocking(move || update_handler.handle_update(meta, data, index)).await?; - Ok(result) + Ok(spawn_blocking(move || update_handler.handle_update(meta, data, index)).await?) } async fn handle_settings(&self, uuid: Uuid) -> IndexResult> { diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 28204f4c0..39de02ef1 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -303,9 +303,13 @@ impl UpdateStore { } None => None, }; + // Process the pending update using the provided user function. - let result = Handle::current() - .block_on(index_handle.update(index_uuid, processing, file))?; + let handle = Handle::current(); + let result = match handle.block_on(index_handle.update(index_uuid, processing.clone(), file)) { + Ok(result) => result, + Err(e) => Err(processing.fail(e.to_string())), + }; // Once the pending update have been successfully processed // we must remove the content from the pending and processing stores and From 1a65eed724fcef7fd5e87cacfd4578400f7bbcad Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 9 Jun 2021 11:52:36 +0200 Subject: [PATCH 367/522] fix index creation bug --- .../src/index_controller/index_actor/store.rs | 7 +++- meilisearch-http/src/index_controller/mod.rs | 3 +- .../update_actor/store/mod.rs | 4 +-- .../index_controller/uuid_resolver/actor.rs | 10 ------ .../uuid_resolver/handle_impl.rs | 9 ----- .../index_controller/uuid_resolver/message.rs | 4 --- .../src/index_controller/uuid_resolver/mod.rs | 1 - .../index_controller/uuid_resolver/store.rs | 34 ++++--------------- 8 files changed, 16 insertions(+), 56 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 11791be48..8f892587d 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -40,6 +40,11 @@ impl MapIndexStore { #[async_trait::async_trait] impl IndexStore for MapIndexStore { async fn create(&self, uuid: Uuid, primary_key: Option) -> IndexResult { + let mut lock = self.index_store.write().await; + + if let Some(index) = lock.get(&uuid) { + return Ok(index.clone()) + } let path = self.path.join(format!("index-{}", uuid)); if path.exists() { return Err(IndexError::IndexAlreadyExists); @@ -57,7 +62,7 @@ impl IndexStore for MapIndexStore { }) .await??; - self.index_store.write().await.insert(uuid, index.clone()); + lock.insert(uuid, index.clone()); Ok(index) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index f562d2185..4d5a52666 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -249,8 +249,9 @@ impl IndexController { ) -> anyhow::Result { let IndexSettings { uid, primary_key } = index_settings; let uid = uid.ok_or_else(|| anyhow::anyhow!("Can't create an index without a uid."))?; - let uuid = self.uuid_resolver.create(uid.clone()).await?; + let uuid = Uuid::new_v4(); let meta = self.index_handle.create_index(uuid, primary_key).await?; + self.uuid_resolver.insert(uid.clone(), uuid).await?; let meta = IndexMetadata { uuid, name: uid.clone(), diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 39de02ef1..331e7b2bb 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -229,7 +229,7 @@ impl UpdateStore { let mut txn = self.env.write_txn()?; let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?; - let meta = Enqueued::new(meta, update_id, content); + let meta = dbg!(Enqueued::new(meta, update_id, content)); self.pending_queue .put(&mut txn, &(global_id, index_uuid, update_id), &meta)?; @@ -280,7 +280,7 @@ impl UpdateStore { ) -> anyhow::Result> { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; - let first_meta = self.pending_queue.first(&rtxn)?; + let first_meta = dbg!(self.pending_queue.first(&rtxn)?); drop(rtxn); // If there is a pending update we process and only keep diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 0211cef25..74158ce04 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -23,9 +23,6 @@ impl UuidResolverActor { loop { match self.inbox.recv().await { - Some(Create { uid: name, ret }) => { - let _ = ret.send(self.handle_create(name).await); - } Some(Get { uid: name, ret }) => { let _ = ret.send(self.handle_get(name).await); } @@ -55,13 +52,6 @@ impl UuidResolverActor { warn!("exiting uuid resolver loop"); } - async fn handle_create(&self, uid: String) -> Result { - if !is_index_uid_valid(&uid) { - return Err(UuidResolverError::BadlyFormatted(uid)); - } - self.store.create_uuid(uid, true).await - } - async fn handle_get(&self, uid: String) -> Result { self.store .get_uuid(uid.clone()) diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index 981beb0f6..af710dd87 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -32,15 +32,6 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - async fn create(&self, name: String) -> anyhow::Result { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Create { uid: name, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - async fn delete(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Delete { uid: name, ret }; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index 2092c67fd..46d9b585f 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -11,10 +11,6 @@ pub enum UuidResolveMsg { uid: String, ret: oneshot::Sender>, }, - Create { - uid: String, - ret: oneshot::Sender>, - }, Delete { uid: String, ret: oneshot::Sender>, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 5bddadf02..3c3b5fd06 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -28,7 +28,6 @@ pub type Result = std::result::Result; pub trait UuidResolverHandle { async fn get(&self, name: String) -> Result; async fn insert(&self, name: String, uuid: Uuid) -> anyhow::Result<()>; - async fn create(&self, name: String) -> anyhow::Result; async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; async fn snapshot(&self, path: PathBuf) -> Result>; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 1d6ada269..5f7c23f97 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -8,7 +8,7 @@ use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use super::{Result, UuidResolverError, UUID_STORE_SIZE}; +use super::{Result, UUID_STORE_SIZE, UuidResolverError}; use crate::helpers::EnvSizer; #[derive(Serialize, Deserialize)] @@ -23,7 +23,6 @@ const UUIDS_DB_PATH: &str = "index_uuids"; pub trait UuidStore: Sized { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return // the uuid otherwise. - async fn create_uuid(&self, uid: String, err: bool) -> Result; async fn get_uuid(&self, uid: String) -> Result>; async fn delete(&self, uid: String) -> Result>; async fn list(&self) -> Result>; @@ -50,27 +49,6 @@ impl HeedUuidStore { Ok(Self { env, db }) } - pub fn create_uuid(&self, name: String, err: bool) -> Result { - let env = self.env.clone(); - let db = self.db; - let mut txn = env.write_txn()?; - match db.get(&txn, &name)? { - Some(uuid) => { - if err { - Err(UuidResolverError::NameAlreadyExist) - } else { - let uuid = Uuid::from_slice(uuid)?; - Ok(uuid) - } - } - None => { - let uuid = Uuid::new_v4(); - db.put(&mut txn, &name, uuid.as_bytes())?; - txn.commit()?; - Ok(uuid) - } - } - } pub fn get_uuid(&self, name: String) -> Result> { let env = self.env.clone(); let db = self.db; @@ -116,6 +94,11 @@ impl HeedUuidStore { let env = self.env.clone(); let db = self.db; let mut txn = env.write_txn()?; + + if db.get(&txn, &name)?.is_some() { + return Err(UuidResolverError::NameAlreadyExist) + } + db.put(&mut txn, &name, uuid.as_bytes())?; txn.commit()?; Ok(()) @@ -205,11 +188,6 @@ impl HeedUuidStore { #[async_trait::async_trait] impl UuidStore for HeedUuidStore { - async fn create_uuid(&self, name: String, err: bool) -> Result { - let this = self.clone(); - tokio::task::spawn_blocking(move || this.create_uuid(name, err)).await? - } - async fn get_uuid(&self, name: String) -> Result> { let this = self.clone(); tokio::task::spawn_blocking(move || this.get_uuid(name)).await? From 2716c1aebb1cf9961c5da181b10b384baf3254ea Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 9 Jun 2021 16:19:45 +0200 Subject: [PATCH 368/522] fix update store lock --- .../src/index_controller/index_actor/store.rs | 2 +- .../index_controller/update_actor/actor.rs | 17 ++- .../update_actor/handle_impl.rs | 66 +++++++-- .../src/index_controller/update_actor/mod.rs | 4 + .../update_actor/store/mod.rs | 138 ++++++++++++------ .../src/index_controller/updates.rs | 2 +- .../index_controller/uuid_resolver/store.rs | 4 +- 7 files changed, 163 insertions(+), 70 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 8f892587d..9b6b057c3 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -43,7 +43,7 @@ impl IndexStore for MapIndexStore { let mut lock = self.index_store.write().await; if let Some(index) = lock.get(&uuid) { - return Ok(index.clone()) + return Ok(index.clone()); } let path = self.path.join(format!("index-{}", uuid)); if path.exists() { diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 7779f2556..76cba7e07 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -1,6 +1,7 @@ use std::collections::HashSet; use std::io::SeekFrom; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; use std::sync::Arc; use log::info; @@ -19,6 +20,7 @@ pub struct UpdateActor { store: Arc, inbox: mpsc::Receiver>, index_handle: I, + must_exit: Arc, } impl UpdateActor @@ -39,14 +41,17 @@ where let mut options = heed::EnvOpenOptions::new(); options.map_size(update_db_size); - let store = UpdateStore::open(options, &path, index_handle.clone())?; + let must_exit = Arc::new(AtomicBool::new(false)); + + let store = UpdateStore::open(options, &path, index_handle.clone(), must_exit.clone())?; std::fs::create_dir_all(path.join("update_files"))?; - assert!(path.exists()); + Ok(Self { path, store, inbox, index_handle, + must_exit, }) } @@ -56,7 +61,13 @@ where info!("Started update actor."); loop { - match self.inbox.recv().await { + let msg = self.inbox.recv().await; + + if self.must_exit.load(std::sync::atomic::Ordering::Relaxed) { + break; + } + + match msg { Some(Update { uuid, meta, diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index cc5ba9757..7844bf855 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -7,7 +7,8 @@ use uuid::Uuid; use crate::index_controller::{IndexActorHandle, UpdateStatus}; use super::{ - PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStoreInfo, + PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateError, UpdateMeta, UpdateMsg, + UpdateStoreInfo, }; #[derive(Clone)] @@ -47,42 +48,72 @@ where async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::ListUpdates { uuid, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") + self.sender + .send(msg) + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)?; + receiver + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)? } async fn update_status(&self, uuid: Uuid, id: u64) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::GetUpdate { uuid, id, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") + self.sender + .send(msg) + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)?; + receiver + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)? } async fn delete(&self, uuid: Uuid) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Delete { uuid, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") + self.sender + .send(msg) + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)?; + receiver + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)? } async fn snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Snapshot { uuids, path, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") + self.sender + .send(msg) + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)?; + receiver + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)? } async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Dump { uuids, path, ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") + self.sender + .send(msg) + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)?; + receiver + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)? } async fn get_info(&self) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::GetInfo { ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") + self.sender + .send(msg) + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)?; + receiver + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)? } async fn update( @@ -98,7 +129,12 @@ where meta, ret, }; - let _ = self.sender.send(msg).await; - receiver.await.expect("update actor killed.") + self.sender + .send(msg) + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)?; + receiver + .await + .map_err(|_| UpdateError::FatalUpdateStoreError)? } } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index ba89eebe3..b854cca70 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -30,6 +30,10 @@ pub enum UpdateError { UnexistingUpdate(u64), #[error("Internal error processing update: {0}")] Internal(String), + #[error( + "Update store was shut down due to a fatal error, please check your logs for more info." + )] + FatalUpdateStoreError, } macro_rules! internal_error { diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 331e7b2bb..7ab854be7 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -3,6 +3,7 @@ pub mod dump; use std::fs::{copy, create_dir_all, remove_file, File}; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::{ collections::{BTreeMap, HashSet}, @@ -98,7 +99,7 @@ pub struct UpdateStore { /// | 16-bytes | 8-bytes | updates: Database>, /// Indicates the current state of the update store, - pub state: Arc, + state: Arc, /// Wake up the loop when a new event occurs. notification_sender: mpsc::Sender<()>, path: PathBuf, @@ -138,6 +139,7 @@ impl UpdateStore { options: EnvOpenOptions, path: impl AsRef, index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static, + must_exit: Arc, ) -> anyhow::Result> { let (update_store, mut notification_receiver) = Self::new(options, path)?; let update_store = Arc::new(update_store); @@ -171,7 +173,11 @@ impl UpdateStore { match res { Ok(Some(_)) => (), Ok(None) => break, - Err(e) => error!("error while processing update: {}", e), + Err(e) => { + error!("Fatal error while processing update that requires the update store to shutdown: {}", e); + must_exit.store(true, Ordering::SeqCst); + break 'outer; + } } } // the ownership on the arc has been taken, we need to exit. @@ -181,6 +187,8 @@ impl UpdateStore { } }); + error!("Update store loop exited."); + Ok(update_store) } @@ -286,63 +294,79 @@ impl UpdateStore { // If there is a pending update we process and only keep // a reader while processing it, not a writer. match first_meta { - Some(((global_id, index_uuid, update_id), mut pending)) => { - let content_path = pending.content.take(); + Some(((global_id, index_uuid, _), mut pending)) => { + let content = pending.content.take(); let processing = pending.processing(); - // Acquire the state lock and set the current state to processing. // txn must *always* be acquired after state lock, or it will dead lock. let state = self.state.write(); state.swap(State::Processing(index_uuid, processing.clone())); - let file = match content_path { - Some(uuid) => { - let path = update_uuid_to_file_path(&self.path, uuid); - let file = File::open(path)?; - Some(file) - } - None => None, - }; + let result = + self.perform_update(content, processing, index_handle, index_uuid, global_id); - // Process the pending update using the provided user function. - let handle = Handle::current(); - let result = match handle.block_on(index_handle.update(index_uuid, processing.clone(), file)) { - Ok(result) => result, - Err(e) => Err(processing.fail(e.to_string())), - }; - - // Once the pending update have been successfully processed - // we must remove the content from the pending and processing stores and - // write the *new* meta to the processed-meta store and commit. - let mut wtxn = self.env.write_txn()?; - self.pending_queue - .delete(&mut wtxn, &(global_id, index_uuid, update_id))?; - - if let Some(uuid) = content_path { - let path = update_uuid_to_file_path(&self.path, uuid); - remove_file(&path)?; - } - - let result = match result { - Ok(res) => res.into(), - Err(res) => res.into(), - }; - - self.updates.remap_key_type::().put( - &mut wtxn, - &(index_uuid, update_id), - &result, - )?; - - wtxn.commit()?; state.swap(State::Idle); - Ok(Some(())) + result } None => Ok(None), } } + fn perform_update( + &self, + content: Option, + processing: Processing, + index_handle: impl IndexActorHandle, + index_uuid: Uuid, + global_id: u64, + ) -> anyhow::Result> { + let content_path = content.map(|uuid| update_uuid_to_file_path(&self.path, uuid)); + let update_id = processing.id(); + + let file = match content_path { + Some(ref path) => { + let file = File::open(path)?; + Some(file) + } + None => None, + }; + + // Process the pending update using the provided user function. + let handle = Handle::current(); + let result = + match handle.block_on(index_handle.update(index_uuid, processing.clone(), file)) { + Ok(result) => result, + Err(e) => Err(processing.fail(e.to_string())), + }; + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.pending_queue + .delete(&mut wtxn, &(global_id, index_uuid, update_id))?; + + let result = match result { + Ok(res) => res.into(), + Err(res) => res.into(), + }; + + self.updates.remap_key_type::().put( + &mut wtxn, + &(index_uuid, update_id), + &result, + )?; + + wtxn.commit()?; + + if let Some(ref path) = content_path { + remove_file(&path)?; + } + + Ok(Some(())) + } + /// List the updates for `index_uuid`. pub fn list(&self, index_uuid: Uuid) -> anyhow::Result> { let mut update_list = BTreeMap::::new(); @@ -561,7 +585,13 @@ mod test { let mut options = EnvOpenOptions::new(); let handle = Arc::new(MockIndexActorHandle::new()); options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir.path(), handle).unwrap(); + let update_store = UpdateStore::open( + options, + dir.path(), + handle, + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); let index1_uuid = Uuid::new_v4(); let index2_uuid = Uuid::new_v4(); @@ -588,7 +618,13 @@ mod test { let mut options = EnvOpenOptions::new(); let handle = Arc::new(MockIndexActorHandle::new()); options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir.path(), handle).unwrap(); + let update_store = UpdateStore::open( + options, + dir.path(), + handle, + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); let meta = UpdateMeta::ClearDocuments; let uuid = Uuid::new_v4(); let store_clone = update_store.clone(); @@ -626,7 +662,13 @@ mod test { let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100); - let store = UpdateStore::open(options, dir.path(), handle.clone()).unwrap(); + let store = UpdateStore::open( + options, + dir.path(), + handle.clone(), + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); // wait a bit for the event loop exit. tokio::time::sleep(std::time::Duration::from_millis(50)).await; diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 303289df3..0dc1c2534 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -3,7 +3,7 @@ use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::index::{Unchecked, Settings}; +use crate::index::{Settings, Unchecked}; pub type UpdateError = String; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 5f7c23f97..bab223bb3 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -8,7 +8,7 @@ use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use super::{Result, UUID_STORE_SIZE, UuidResolverError}; +use super::{Result, UuidResolverError, UUID_STORE_SIZE}; use crate::helpers::EnvSizer; #[derive(Serialize, Deserialize)] @@ -96,7 +96,7 @@ impl HeedUuidStore { let mut txn = env.write_txn()?; if db.get(&txn, &name)?.is_some() { - return Err(UuidResolverError::NameAlreadyExist) + return Err(UuidResolverError::NameAlreadyExist); } db.put(&mut txn, &name, uuid.as_bytes())?; From 99551fc21b72b1475cf3f732d5096bc6879e9d3a Mon Sep 17 00:00:00 2001 From: Irevoire Date: Wed, 9 Jun 2021 17:10:10 +0200 Subject: [PATCH 369/522] fix encoding bug --- meilisearch-http/src/index/update_handler.rs | 2 +- meilisearch-http/src/index/updates.rs | 8 ++--- meilisearch-http/src/index_controller/mod.rs | 13 ++----- .../index_controller/update_actor/actor.rs | 7 ++-- .../update_actor/store/codec.rs | 4 +-- .../update_actor/store/dump.rs | 1 - .../update_actor/store/mod.rs | 34 +++++++++---------- .../src/index_controller/updates.rs | 4 ++- .../tests/documents/delete_documents.rs | 2 +- 9 files changed, 34 insertions(+), 41 deletions(-) diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs index 8a127168e..13e516d3c 100644 --- a/meilisearch-http/src/index/update_handler.rs +++ b/meilisearch-http/src/index/update_handler.rs @@ -81,7 +81,7 @@ impl UpdateHandler { primary_key.as_deref(), ), ClearDocuments => index.clear_documents(update_builder), - DeleteDocuments => index.delete_documents(content, update_builder), + DeleteDocuments { documents } => index.delete_documents(documents.to_vec(), update_builder), Settings(settings) => index.update_settings(&settings.clone().check(), update_builder), }; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index b4869fa42..750e1f275 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -298,18 +298,14 @@ impl Index { pub fn delete_documents( &self, - document_ids: Option, + document_ids: Vec, update_builder: UpdateBuilder, ) -> anyhow::Result { - let ids = match document_ids { - Some(reader) => serde_json::from_reader(reader)?, - None => Vec::::new(), - }; let mut txn = self.write_txn()?; let mut builder = update_builder.delete_documents(&mut txn, self)?; // We ignore unexisting document ids - ids.iter().for_each(|id| { + document_ids.iter().for_each(|id| { builder.delete_external_id(id); }); diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 4d5a52666..3c46e48f6 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -200,18 +200,11 @@ impl IndexController { pub async fn delete_documents( &self, uid: String, - document_ids: Vec, + documents: Vec, ) -> anyhow::Result { let uuid = self.uuid_resolver.get(uid).await?; - let meta = UpdateMeta::DeleteDocuments; - let (sender, receiver) = mpsc::channel(10); - - tokio::task::spawn(async move { - let json = serde_json::to_vec(&document_ids).unwrap(); - let bytes = Bytes::from(json); - let _ = sender.send(Ok(bytes)).await; - }); - + let meta = UpdateMeta::DeleteDocuments { documents }; + let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; Ok(status) } diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 76cba7e07..c74cf11f5 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -106,7 +106,7 @@ where mut payload: mpsc::Receiver>, ) -> Result { let file_path = match meta { - UpdateMeta::DocumentsAddition { .. } | UpdateMeta::DeleteDocuments => { + UpdateMeta::DocumentsAddition { .. } => { let update_file_id = uuid::Uuid::new_v4(); let path = self .path @@ -181,10 +181,13 @@ where async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { let store = self.store.clone(); + tokio::task::spawn_blocking(move || { let result = store .meta(uuid, id)? .ok_or(UpdateError::UnexistingUpdate(id))?; - Ok(result) + Ok(result) + }) + .await? } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { diff --git a/meilisearch-http/src/index_controller/update_actor/store/codec.rs b/meilisearch-http/src/index_controller/update_actor/store/codec.rs index e07b52eec..2c7068f88 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/codec.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/codec.rs @@ -75,10 +75,10 @@ impl<'a> BytesDecode<'a> for UpdateKeyCodec { type DItem = (Uuid, u64); fn bytes_decode(bytes: &'a [u8]) -> Option { - let uuid_bytes = bytes.get(0..size_of::())?.try_into().ok()?; + let uuid_bytes = dbg!(bytes.get(0..size_of::())?.try_into().ok())?; let uuid = Uuid::from_bytes(uuid_bytes); - let update_id_bytes = bytes.get(size_of::()..)?.try_into().ok()?; + let update_id_bytes = dbg!(bytes.get(size_of::()..)?.try_into().ok())?; let update_id = u64::from_be_bytes(update_id_bytes); Some((uuid, update_id)) diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index 8f947e459..4be5e27a7 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -108,7 +108,6 @@ impl UpdateStore { let updates = self .updates .iter(txn)? - .remap_key_type::() .lazily_decode_data(); for update in updates { diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 7ab854be7..75b1c5b15 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -97,7 +97,7 @@ pub struct UpdateStore { /// The keys are built as follow: /// | Uuid | id | /// | 16-bytes | 8-bytes | - updates: Database>, + updates: Database>, /// Indicates the current state of the update store, state: Arc, /// Wake up the loop when a new event occurs. @@ -244,6 +244,8 @@ impl UpdateStore { txn.commit()?; + dbg!("here"); + self.notification_sender .blocking_send(()) .expect("Update store loop exited."); @@ -269,7 +271,7 @@ impl UpdateStore { } _ => { let _update_id = self.next_update_id_raw(wtxn, index_uuid)?; - self.updates.remap_key_type::().put( + self.updates.put( wtxn, &(index_uuid, update.id()), &update, @@ -324,6 +326,8 @@ impl UpdateStore { let content_path = content.map(|uuid| update_uuid_to_file_path(&self.path, uuid)); let update_id = processing.id(); + dbg!(&processing); + let file = match content_path { Some(ref path) => { let file = File::open(path)?; @@ -352,7 +356,7 @@ impl UpdateStore { Err(res) => res.into(), }; - self.updates.remap_key_type::().put( + self.updates.put( &mut wtxn, &(index_uuid, update_id), &result, @@ -381,7 +385,11 @@ impl UpdateStore { } } - let updates = self.updates.prefix_iter(&txn, index_uuid.as_bytes())?; + let updates = self + .updates + .remap_key_type::() + .prefix_iter(&txn, index_uuid.as_bytes())?; + for entry in updates { let (_, update) = entry?; update_list.insert(update.id(), update); @@ -412,26 +420,19 @@ impl UpdateStore { let txn = self.env.read_txn()?; // Else, check if it is in the updates database: - let update = self - .updates - .remap_key_type::() - .get(&txn, &(index_uuid, update_id))?; + let update = dbg!(self.updates.get(&txn, &(index_uuid, update_id)))?; if let Some(update) = update { return Ok(Some(update)); } // If nothing was found yet, we resolve to iterate over the pending queue. - let pendings = self - .pending_queue - .remap_key_type::() - .iter(&txn)? - .lazily_decode_data(); + let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); for entry in pendings { - let ((uuid, id), pending) = entry?; + let ((_, uuid, id), pending) = entry?; if uuid == index_uuid && id == update_id { - return Ok(Some(pending.decode()?.into())); + return Ok(Some(dbg!(pending.decode())?.into())); } } @@ -461,6 +462,7 @@ impl UpdateStore { let mut updates = self .updates + .remap_key_type::() .prefix_iter_mut(&mut txn, index_uuid.as_bytes())? .lazily_decode_data(); @@ -707,7 +709,6 @@ mod test { assert!(store.pending_queue.first(&txn).unwrap().is_none()); let update = store .updates - .remap_key_type::() .get(&txn, &(uuid, 0)) .unwrap() .unwrap(); @@ -715,7 +716,6 @@ mod test { assert!(matches!(update, UpdateStatus::Processed(_))); let update = store .updates - .remap_key_type::() .get(&txn, &(uuid, 1)) .unwrap() .unwrap(); diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 0dc1c2534..86b33e3f2 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -23,7 +23,9 @@ pub enum UpdateMeta { primary_key: Option, }, ClearDocuments, - DeleteDocuments, + DeleteDocuments { + documents: Vec + }, Settings(Settings), } diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs index b69b4c11f..d9b97d68d 100644 --- a/meilisearch-http/tests/documents/delete_documents.rs +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -114,7 +114,7 @@ async fn delete_no_document_batch() { index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; index.wait_update_id(0).await; let (_response, code) = index.delete_batch(vec![]).await; - assert_eq!(code, 202); + assert_eq!(code, 202, "{}", _response); let _update = index.wait_update_id(1).await; let (response, code) = index From 2d19b78dd8941631b853f3e0ffd7a1646d77a8ae Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 10 Jun 2021 12:03:16 +0200 Subject: [PATCH 370/522] fix stats test --- meilisearch-http/tests/index/stats.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs index e1d8bd211..d32c06d2b 100644 --- a/meilisearch-http/tests/index/stats.rs +++ b/meilisearch-http/tests/index/stats.rs @@ -35,11 +35,6 @@ async fn stats() { assert_eq!(code, 202); assert_eq!(response["updateId"], 0); - let (response, code) = index.stats().await; - - assert_eq!(code, 200); - assert_eq!(response["isIndexing"], true); - index.wait_update_id(0).await; let (response, code) = index.stats().await; From 20e1caef470316f5ab2f493aa027150f869f4b75 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Thu, 10 Jun 2021 14:53:34 +0200 Subject: [PATCH 371/522] makes clippy happy --- .../src/index_controller/update_actor/store/dump.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index 4be5e27a7..e7f36a2a1 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -9,8 +9,7 @@ use heed::{EnvOpenOptions, RoTxn}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use super::UpdateStore; -use super::{codec::UpdateKeyCodec, State}; +use super::{State, UpdateStore}; use crate::index_controller::{ index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued, UpdateStatus, @@ -105,10 +104,7 @@ impl UpdateStore { uuids: &HashSet, mut file: &mut File, ) -> anyhow::Result<()> { - let updates = self - .updates - .iter(txn)? - .lazily_decode_data(); + let updates = self.updates.iter(txn)?.lazily_decode_data(); for update in updates { let ((uuid, _), data) = update?; From 592fcbc71fcb03533f40e98738411f79d9c36b18 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 10 Jun 2021 12:03:16 +0200 Subject: [PATCH 372/522] fix stats test --- meilisearch-http/tests/stats/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index ef90dcf7f..f931d5066 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -53,13 +53,12 @@ async fn stats() { ]); let (response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 202); + assert_eq!(code, 202, "{}", response); assert_eq!(response["updateId"], 0); let (response, code) = server.stats().await; - assert_eq!(code, 200); - assert_eq!(response["indexes"]["test"]["isIndexing"], true); + assert_eq!(code, 200, "{}", response); index.wait_update_id(0).await; From eb7616ca0fd4e013462976b9dc07e15d5cf5963f Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 10 Jun 2021 15:32:45 +0200 Subject: [PATCH 373/522] remove dbg --- .../src/index_controller/update_actor/store/codec.rs | 4 ++-- .../src/index_controller/update_actor/store/mod.rs | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/store/codec.rs b/meilisearch-http/src/index_controller/update_actor/store/codec.rs index 2c7068f88..e07b52eec 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/codec.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/codec.rs @@ -75,10 +75,10 @@ impl<'a> BytesDecode<'a> for UpdateKeyCodec { type DItem = (Uuid, u64); fn bytes_decode(bytes: &'a [u8]) -> Option { - let uuid_bytes = dbg!(bytes.get(0..size_of::())?.try_into().ok())?; + let uuid_bytes = bytes.get(0..size_of::())?.try_into().ok()?; let uuid = Uuid::from_bytes(uuid_bytes); - let update_id_bytes = dbg!(bytes.get(size_of::()..)?.try_into().ok())?; + let update_id_bytes = bytes.get(size_of::()..)?.try_into().ok()?; let update_id = u64::from_be_bytes(update_id_bytes); Some((uuid, update_id)) diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 75b1c5b15..fc5c44568 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -237,15 +237,13 @@ impl UpdateStore { let mut txn = self.env.write_txn()?; let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?; - let meta = dbg!(Enqueued::new(meta, update_id, content)); + let meta = Enqueued::new(meta, update_id, content); self.pending_queue .put(&mut txn, &(global_id, index_uuid, update_id), &meta)?; txn.commit()?; - dbg!("here"); - self.notification_sender .blocking_send(()) .expect("Update store loop exited."); @@ -290,7 +288,7 @@ impl UpdateStore { ) -> anyhow::Result> { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; - let first_meta = dbg!(self.pending_queue.first(&rtxn)?); + let first_meta = self.pending_queue.first(&rtxn)?; drop(rtxn); // If there is a pending update we process and only keep @@ -326,8 +324,6 @@ impl UpdateStore { let content_path = content.map(|uuid| update_uuid_to_file_path(&self.path, uuid)); let update_id = processing.id(); - dbg!(&processing); - let file = match content_path { Some(ref path) => { let file = File::open(path)?; @@ -420,7 +416,7 @@ impl UpdateStore { let txn = self.env.read_txn()?; // Else, check if it is in the updates database: - let update = dbg!(self.updates.get(&txn, &(index_uuid, update_id)))?; + let update = self.updates.get(&txn, &(index_uuid, update_id))?; if let Some(update) = update { return Ok(Some(update)); @@ -432,7 +428,7 @@ impl UpdateStore { for entry in pendings { let ((_, uuid, id), pending) = entry?; if uuid == index_uuid && id == update_id { - return Ok(Some(dbg!(pending.decode())?.into())); + return Ok(Some(pending.decode()?.into())); } } From 3ef0830c5dcde12066c9a489b9c441e6717c541f Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 10 Jun 2021 15:55:44 +0200 Subject: [PATCH 374/522] review changes --- meilisearch-http/src/index/update_handler.rs | 2 +- meilisearch-http/src/index/updates.rs | 2 +- meilisearch-http/src/index_controller/index_actor/store.rs | 2 ++ meilisearch-http/src/index_controller/mod.rs | 2 +- .../src/index_controller/update_actor/store/mod.rs | 6 +++--- meilisearch-http/src/index_controller/updates.rs | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs index 13e516d3c..63a074abb 100644 --- a/meilisearch-http/src/index/update_handler.rs +++ b/meilisearch-http/src/index/update_handler.rs @@ -81,7 +81,7 @@ impl UpdateHandler { primary_key.as_deref(), ), ClearDocuments => index.clear_documents(update_builder), - DeleteDocuments { documents } => index.delete_documents(documents.to_vec(), update_builder), + DeleteDocuments { ids } => index.delete_documents(ids, update_builder), Settings(settings) => index.update_settings(&settings.clone().check(), update_builder), }; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 750e1f275..9ed4fe49e 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -298,7 +298,7 @@ impl Index { pub fn delete_documents( &self, - document_ids: Vec, + document_ids: &[String], update_builder: UpdateBuilder, ) -> anyhow::Result { let mut txn = self.write_txn()?; diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 9b6b057c3..1646821d8 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -40,6 +40,8 @@ impl MapIndexStore { #[async_trait::async_trait] impl IndexStore for MapIndexStore { async fn create(&self, uuid: Uuid, primary_key: Option) -> IndexResult { + // We need to keep the lock until we are sure the db file has been opened correclty, to + // ensure that another db is not created at the same time. let mut lock = self.index_store.write().await; if let Some(index) = lock.get(&uuid) { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 3c46e48f6..0c801558b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -203,7 +203,7 @@ impl IndexController { documents: Vec, ) -> anyhow::Result { let uuid = self.uuid_resolver.get(uid).await?; - let meta = UpdateMeta::DeleteDocuments { documents }; + let meta = UpdateMeta::DeleteDocuments { ids: documents }; let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; Ok(status) diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index fc5c44568..e7b719fc9 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -174,7 +174,7 @@ impl UpdateStore { Ok(Some(_)) => (), Ok(None) => break, Err(e) => { - error!("Fatal error while processing update that requires the update store to shutdown: {}", e); + error!("Fatal error while processing an update that requires the update store to shutdown: {}", e); must_exit.store(true, Ordering::SeqCst); break 'outer; } @@ -185,9 +185,9 @@ impl UpdateStore { } } } - }); - error!("Update store loop exited."); + error!("Update store loop exited."); + }); Ok(update_store) } diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 86b33e3f2..ea2ffb80d 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -24,7 +24,7 @@ pub enum UpdateMeta { }, ClearDocuments, DeleteDocuments { - documents: Vec + ids: Vec }, Settings(Settings), } From 7312c1366554310662f2176a5a507733da3ce0a4 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Thu, 10 Jun 2021 16:53:30 +0200 Subject: [PATCH 375/522] add a github action to run cargo-flaky 1000 times --- .github/workflows/flaky.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/flaky.yml diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml new file mode 100644 index 000000000..5ecd7a77f --- /dev/null +++ b/.github/workflows/flaky.yml @@ -0,0 +1,15 @@ +name: "Look for flaky tests" +on: + schedule: + - cron: "* */12 * * */5" # every friday at 12PM + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: "install cargo-flaky" + run: cargo install cargo-flaky + - name: "Run cargo flaky 1000 times" + run: cargo flaky -i 1000 From 41220a7f963c43e7b4c2a88c94ef4b2fcfc429d8 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Thu, 10 Jun 2021 17:02:06 +0200 Subject: [PATCH 376/522] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clémentine Urquizar --- .github/workflows/flaky.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml index 5ecd7a77f..46bf76042 100644 --- a/.github/workflows/flaky.yml +++ b/.github/workflows/flaky.yml @@ -1,15 +1,15 @@ -name: "Look for flaky tests" +name: Look for flaky tests on: schedule: - cron: "* */12 * * */5" # every friday at 12PM jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v1 - - name: "install cargo-flaky" + - uses: actions/checkout@v2 + - name: Install cargo-flaky run: cargo install cargo-flaky - - name: "Run cargo flaky 1000 times" + - name: Run cargo flaky 1000 times run: cargo flaky -i 1000 From efc1225cd86f976eb0717d022a782c1a9464cdc2 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Thu, 10 Jun 2021 17:07:23 +0200 Subject: [PATCH 377/522] Update .github/workflows/flaky.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clémentine Urquizar --- .github/workflows/flaky.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml index 46bf76042..e85e34341 100644 --- a/.github/workflows/flaky.yml +++ b/.github/workflows/flaky.yml @@ -4,7 +4,7 @@ on: - cron: "* */12 * * */5" # every friday at 12PM jobs: - build: + flaky: runs-on: ubuntu-18.04 steps: From 51105d3b1ce7cbc398325927dfd436d871e1ac14 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Thu, 10 Jun 2021 17:12:07 +0200 Subject: [PATCH 378/522] run the tests in release mode --- .github/workflows/flaky.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml index e85e34341..5f7fbac6a 100644 --- a/.github/workflows/flaky.yml +++ b/.github/workflows/flaky.yml @@ -12,4 +12,4 @@ jobs: - name: Install cargo-flaky run: cargo install cargo-flaky - name: Run cargo flaky 1000 times - run: cargo flaky -i 1000 + run: cargo flaky -i 1000 --release From e8bd5ea4e051cdc04135a982b9a48a4e4b85c6e0 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 29 Apr 2021 19:31:58 +0200 Subject: [PATCH 379/522] convert UpdateStatus to legacy meilisearch format --- .../src/index_controller/updates.rs | 28 ++- meilisearch-http/src/routes/index.rs | 16 +- meilisearch-http/src/routes/mod.rs | 193 ++++++++++++++++++ 3 files changed, 231 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index ea2ffb80d..c2f7db30e 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -84,6 +84,10 @@ impl Processed { pub fn id(&self) -> u64 { self.from.id() } + + pub fn meta(&self) -> &UpdateMeta { + self.from.meta() + } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -132,21 +136,29 @@ impl Aborted { pub fn id(&self) -> u64 { self.from.id() } + + pub fn meta(&self) -> &UpdateMeta { + self.from.meta() + } } #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct Failed { #[serde(flatten)] - from: Processing, - error: UpdateError, - failed_at: DateTime, + pub from: Processing, + pub error: UpdateError, + pub failed_at: DateTime, } impl Failed { pub fn id(&self) -> u64 { self.from.id() } + + pub fn meta(&self) -> &UpdateMeta { + self.from.meta() + } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -170,6 +182,16 @@ impl UpdateStatus { } } + pub fn meta(&self) -> &UpdateMeta { + match self { + UpdateStatus::Processing(u) => u.meta(), + UpdateStatus::Enqueued(u) => u.meta(), + UpdateStatus::Processed(u) => u.meta(), + UpdateStatus::Aborted(u) => u.meta(), + UpdateStatus::Failed(u) => u.meta(), + } + } + pub fn processed(&self) -> Option<&Processed> { match self { UpdateStatus::Processed(p) => Some(p), diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 4dfe90abf..efe2ef711 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use crate::error::ResponseError; use crate::helpers::Authentication; -use crate::routes::IndexParam; +use super::{UpdateStatusResponse, IndexParam}; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { @@ -129,7 +129,10 @@ async fn get_update_status( .get_update_status(params.index_uid, params.update_id) .await; match result { - Ok(meta) => Ok(HttpResponse::Ok().json(meta)), + Ok(meta) => { + let meta = UpdateStatusResponse::from(meta); + Ok(HttpResponse::Ok().json(meta)) + }, Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } @@ -143,7 +146,14 @@ async fn get_all_updates_status( ) -> Result { let result = data.get_updates_status(path.into_inner().index_uid).await; match result { - Ok(metas) => Ok(HttpResponse::Ok().json(metas)), + Ok(metas) => { + let metas = metas + .into_iter() + .map(UpdateStatusResponse::from) + .collect::>(); + + Ok(HttpResponse::Ok().json(metas)) + }, Err(e) => { Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) } diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index a550064ba..71a62c457 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,6 +1,12 @@ +use std::time::Duration; + use actix_web::{get, HttpResponse}; +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use crate::index::Settings; +use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus}; + pub mod document; pub mod dump; pub mod health; @@ -11,6 +17,193 @@ pub mod settings; pub mod stats; pub mod synonym; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "name")] +pub enum UpdateType { + ClearAll, + Customs, + DocumentsAddition { + #[serde(skip_serializing_if = "Option::is_none")] + number: Option + }, + DocumentsPartial { + #[serde(skip_serializing_if = "Option::is_none")] + number: Option + }, + DocumentsDeletion { + #[serde(skip_serializing_if = "Option::is_none")] + number: Option + }, + Settings { settings: Settings }, +} + +impl From<&UpdateStatus> for UpdateType { + fn from(other: &UpdateStatus) -> Self { + use milli::update::IndexDocumentsMethod::*; + + match other.meta() { + UpdateMeta::DocumentsAddition { method, .. } => { + let number = match other { + UpdateStatus::Processed(processed) => match processed.success { + UpdateResult::DocumentsAddition(ref addition) => { + Some(addition.nb_documents) + } + _ => None, + }, + _ => None, + }; + + match method { + ReplaceDocuments => UpdateType::DocumentsAddition { number }, + UpdateDocuments => UpdateType::DocumentsPartial { number }, + _ => unreachable!(), + } + } + UpdateMeta::ClearDocuments => UpdateType::ClearAll, + UpdateMeta::DeleteDocuments => { + let number = match other { + UpdateStatus::Processed(processed) => match processed.success { + UpdateResult::DocumentDeletion { deleted } => Some(deleted as usize), + _ => None, + }, + _ => None, + }; + UpdateType::DocumentsDeletion { number } + } + UpdateMeta::Settings(settings) => UpdateType::Settings { + settings: settings.clone(), + }, + _ => unreachable!(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProcessedUpdateResult { + pub update_id: u64, + #[serde(rename = "type")] + pub update_type: UpdateType, + pub duration: f64, // in seconds + pub enqueued_at: DateTime, + pub processed_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FailedUpdateResult { + pub update_id: u64, + #[serde(rename = "type")] + pub update_type: UpdateType, + pub error: String, + pub error_type: String, + pub error_code: String, + pub error_link: String, + pub duration: f64, // in seconds + pub enqueued_at: DateTime, + pub processed_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EnqueuedUpdateResult { + pub update_id: u64, + #[serde(rename = "type")] + pub update_type: UpdateType, + pub enqueued_at: DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub started_processing_at: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "status")] +pub enum UpdateStatusResponse { + Enqueued { + #[serde(flatten)] + content: EnqueuedUpdateResult, + }, + Processing { + #[serde(flatten)] + content: EnqueuedUpdateResult, + }, + Failed { + #[serde(flatten)] + content: FailedUpdateResult, + }, + Processed { + #[serde(flatten)] + content: ProcessedUpdateResult, + }, +} + +impl From for UpdateStatusResponse { + fn from(other: UpdateStatus) -> Self { + let update_type = UpdateType::from(&other); + + match other { + UpdateStatus::Processing(processing) => { + let content = EnqueuedUpdateResult { + update_id: processing.id(), + update_type, + enqueued_at: processing.from.enqueued_at, + started_processing_at: Some(processing.started_processing_at), + }; + UpdateStatusResponse::Processing { content } + } + UpdateStatus::Enqueued(enqueued) => { + let content = EnqueuedUpdateResult { + update_id: enqueued.id(), + update_type, + enqueued_at: enqueued.enqueued_at, + started_processing_at: None, + }; + UpdateStatusResponse::Enqueued { content } + } + UpdateStatus::Processed(processed) => { + let duration = processed + .processed_at + .signed_duration_since(processed.from.started_processing_at) + .num_milliseconds(); + + // necessary since chrono::duration don't expose a f64 secs method. + let duration = Duration::from_millis(duration as u64).as_secs_f64(); + + let content = ProcessedUpdateResult { + update_id: processed.id(), + update_type, + duration, + enqueued_at: processed.from.from.enqueued_at, + processed_at: processed.processed_at, + }; + UpdateStatusResponse::Processed { content } + } + UpdateStatus::Aborted(_) => unreachable!(), + UpdateStatus::Failed(failed) => { + let duration = failed + .failed_at + .signed_duration_since(failed.from.started_processing_at) + .num_milliseconds(); + + // necessary since chrono::duration don't expose a f64 secs method. + let duration = Duration::from_millis(duration as u64).as_secs_f64(); + + let content = FailedUpdateResult { + update_id: failed.id(), + update_type, + error: failed.error, + error_type: String::from("todo"), + error_code: String::from("todo"), + error_link: String::from("todo"), + duration, + enqueued_at: failed.from.from.enqueued_at, + processed_at: failed.failed_at, + }; + UpdateStatusResponse::Failed { content } + } + } + } +} + #[derive(Deserialize)] pub struct IndexParam { index_uid: String, From 92d954ddfe653804e45cf49770463f48365f7a9f Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 14 Jun 2021 10:48:59 +0200 Subject: [PATCH 380/522] Fix the cron syntax to effectively run the test once every friday --- .github/workflows/flaky.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml index 5f7fbac6a..88c8ee6ca 100644 --- a/.github/workflows/flaky.yml +++ b/.github/workflows/flaky.yml @@ -1,7 +1,7 @@ name: Look for flaky tests on: schedule: - - cron: "* */12 * * */5" # every friday at 12PM + - cron: "0 12 * * FRI" # every friday at 12:00PM jobs: flaky: From 0f767e374325818d4906c2f5378407e4a4a075a8 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 2 Jun 2021 17:45:28 +0200 Subject: [PATCH 381/522] conccurrent update run loop --- Cargo.lock | 483 +++++++++--------- .../index_controller/update_actor/actor.rs | 86 ++-- 2 files changed, 302 insertions(+), 267 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8546e3bc..941f1151f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "actix-codec" version = "0.4.0" @@ -43,7 +41,7 @@ dependencies = [ "actix-service", "actix-tls", "actix-utils", - "ahash 0.7.2", + "ahash 0.7.4", "base64", "bitflags", "brotli2", @@ -70,20 +68,20 @@ dependencies = [ "rand 0.8.3", "regex", "serde", - "sha-1 0.9.4", + "sha-1 0.9.6", "smallvec", - "time 0.2.26", + "time 0.2.27", "tokio", ] [[package]] name = "actix-macros" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" +checksum = "c2f86cd6857c135e6e9fe57b1619a88d1f94a7df34c00e11fe13e64fd3438837" dependencies = [ "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -183,7 +181,7 @@ dependencies = [ "actix-tls", "actix-utils", "actix-web-codegen", - "ahash 0.7.2", + "ahash 0.7.4", "bytes 1.0.1", "cookie", "derive_more", @@ -203,7 +201,7 @@ dependencies = [ "serde_urlencoded", "smallvec", "socket2", - "time 0.2.26", + "time 0.2.27", "url", ] @@ -213,9 +211,9 @@ version = "0.5.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -234,9 +232,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.14.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" +checksum = "e7a2e47a1fbe209ee101dd6d61285226744c6c8d3c21c8dc878ba6cb9f467f3a" dependencies = [ "gimli", ] @@ -255,11 +253,11 @@ checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" [[package]] name = "ahash" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", "once_cell", "version_check", ] @@ -284,15 +282,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" +checksum = "15af2628f6890fe2609a3b91bef4c83450512802e59489f9c1cb1fa5df064a61" [[package]] name = "arc-swap" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73" +checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820" [[package]] name = "assert-json-diff" @@ -305,9 +303,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a26cb53174ddd320edfff199a853f93d571f48eeb4dde75e67a9a3dbb7b7e5e" +checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" dependencies = [ "async-stream-impl", "futures-core", @@ -315,13 +313,13 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340" +checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -330,9 +328,9 @@ version = "0.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -354,9 +352,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.58" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88fb5a785d6b44fd9d6700935608639af1b8356de1e55d5f7c2740f4faa15d82" +checksum = "b7815ea54e4d821e791162e078acbebfd6d8c8939cd559c9335dceb1c8ca7282" dependencies = [ "addr2line", "cc", @@ -458,9 +456,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" [[package]] name = "byte-tools" @@ -470,18 +468,18 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byte-unit" -version = "4.0.11" +version = "4.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d98e67b09c0321733bef2f3b879832afa6197e9ea58f32e72c316df2ffe743" +checksum = "063197e6eb4b775b64160dedde7a0986bb2836cce140e9492e9e96f28e18bcd8" dependencies = [ "utf8-width", ] [[package]] name = "bytemuck" -version = "1.5.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" +checksum = "9966d2ab714d0f785dbac0a0396251a35280aeb42413281617d0209ab4898435" [[package]] name = "byteorder" @@ -512,9 +510,9 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.3.3" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" dependencies = [ "bzip2-sys", "libc", @@ -522,9 +520,9 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.10+1.0.8" +version = "0.1.11+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17fa3d1ac1ca21c5c4e36a97f3c3eb25084576f6fc47bf0139c1123434216c6c" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" dependencies = [ "cc", "libc", @@ -533,9 +531,9 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d94e66797729c3a52b74980ec7992a9399aace3044bbcde9ce6bb98926abb673" +checksum = "9c3596addfb02dcdc06f5252ddda9f3785f9230f5827fb4284645240fa05ad92" dependencies = [ "serde", "serde_derive", @@ -544,9 +542,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" dependencies = [ "jobserver", ] @@ -622,9 +620,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" +checksum = "f92cfa0fd5690b3cf8c1ef2cabbd9b7ef22fa53cf5e1f92b05103f6d5d1cf6e7" [[package]] name = "convert_case" @@ -639,7 +637,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627" dependencies = [ "percent-encoding", - "time 0.2.26", + "time 0.2.27", "version_check", ] @@ -650,10 +648,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" [[package]] -name = "cpuid-bool" -version = "0.1.2" +name = "cpufeatures" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +checksum = "ed00c67cb5d0a7d64a44f6ad2668db7e7530311dd53ea79bcd4fb022c64911c8" +dependencies = [ + "libc", +] [[package]] name = "crc32fast" @@ -671,7 +672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", ] [[package]] @@ -682,17 +683,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", ] [[package]] name = "crossbeam-epoch" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52fb27eab85b17fbb9f6fd667089e07d6a2eb8743d02639ee7f6a7a7729c9c94" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", "lazy_static", "memoffset", "scopeguard", @@ -719,11 +720,10 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" dependencies = [ - "autocfg", "cfg-if 1.0.0", "lazy_static", ] @@ -762,21 +762,21 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.13" +version = "0.99.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b1b72f1263f214c0f823371768776c4f5841b942c9883aa8e5ec584fd0ba6" +checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320" dependencies = [ "convert_case", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] name = "deunicode" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2044dd33b8b682eedf68271c913235cbefdaf17bf842310be1389c21d6e5945c" +checksum = "7f37775d639f64aa16389eede0cbe6a70f56df4609d50d8b6858690d5d7bf8f2" [[package]] name = "difference" @@ -831,9 +831,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" dependencies = [ "atty", "humantime", @@ -911,9 +911,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "fst" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" [[package]] name = "fuchsia-cprng" @@ -923,9 +923,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "futures" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" dependencies = [ "futures-channel", "futures-core", @@ -938,9 +938,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" dependencies = [ "futures-core", "futures-sink", @@ -948,15 +948,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" [[package]] name = "futures-executor" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" +checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" dependencies = [ "futures-core", "futures-task", @@ -965,40 +965,42 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" [[package]] name = "futures-macro" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ + "autocfg", "proc-macro-hack", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] name = "futures-sink" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" [[package]] name = "futures-task" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" [[package]] name = "futures-util" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ + "autocfg", "futures-channel", "futures-core", "futures-io", @@ -1054,9 +1056,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1065,9 +1067,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" [[package]] name = "glob" @@ -1132,9 +1134,9 @@ checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" [[package]] name = "heck" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" dependencies = [ "unicode-segmentation", ] @@ -1216,9 +1218,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" dependencies = [ "bytes 1.0.1", "http", @@ -1227,9 +1229,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" +checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" @@ -1239,9 +1241,9 @@ checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] name = "httpdate" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" [[package]] name = "human_format" @@ -1257,9 +1259,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" +checksum = "07d6baa1b441335f3ce5098ac421fb6547c46dda735ca1bc6d0153c838f9dd83" dependencies = [ "bytes 1.0.1", "futures-channel", @@ -1269,9 +1271,9 @@ dependencies = [ "http", "http-body", "httparse", - "httpdate 1.0.0", + "httpdate 1.0.1", "itoa", - "pin-project", + "pin-project-lite", "socket2", "tokio", "tower-service", @@ -1342,9 +1344,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf" dependencies = [ "either", ] @@ -1378,9 +1380,9 @@ dependencies = [ [[package]] name = "jieba-rs" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31a7e00da0a90e0de5d8dd1193b0216f5590a24ed633ae701ac50ca94467ef07" +checksum = "fea3b3172a80f9958abc3b9a637e4e311cd696dc6813440e5cc929b8a5311055" dependencies = [ "cedarwood", "fxhash", @@ -1402,9 +1404,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.50" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" +checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" dependencies = [ "wasm-bindgen", ] @@ -1423,18 +1425,18 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "levenshtein_automata" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" dependencies = [ "fst", ] [[package]] name = "libc" -version = "0.2.94" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" [[package]] name = "linked-hash-map" @@ -1573,7 +1575,7 @@ dependencies = [ "hex", "http", "indexmap", - "itertools 0.10.0", + "itertools 0.10.1", "jemallocator", "log", "main_error", @@ -1598,7 +1600,7 @@ dependencies = [ "serde", "serde_json", "serde_url_params", - "sha-1 0.9.4", + "sha-1 0.9.6", "sha2", "siphasher", "slice-group-by", @@ -1649,9 +1651,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ "autocfg", ] @@ -1674,7 +1676,7 @@ dependencies = [ "grenad", "heed", "human_format", - "itertools 0.10.0", + "itertools 0.10.1", "levenshtein_automata", "linked-hash-map", "log", @@ -1769,9 +1771,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd4234635bca06fc96c7368d038061e0aae1b00a764dc817e900dc974e3deea" dependencies = [ "cfg-if 1.0.0", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -1832,9 +1834,12 @@ dependencies = [ [[package]] name = "object" -version = "0.23.0" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" +checksum = "a38f2be3697a57b4060074ff41b44c16870d916ad7877c17696e063257482bc7" +dependencies = [ + "memchr", +] [[package]] name = "obkv" @@ -1844,9 +1849,9 @@ checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" [[package]] name = "once_cell" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" [[package]] name = "opaque-debug" @@ -1862,9 +1867,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "ordered-float" -version = "2.2.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8919aecb97e5ee9aceef27e24f39c46b11831130f4a6b7b091ec5de0de12" +checksum = "f100fcfb41e5385e0991f74981732049f9b896821542a219420491046baafdc2" dependencies = [ "num-traits", ] @@ -1972,9 +1977,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "pest_meta", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -2041,9 +2046,9 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -2106,9 +2111,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", "version_check", ] @@ -2118,7 +2123,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", "version_check", ] @@ -2146,9 +2151,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" dependencies = [ "unicode-xid 0.2.2", ] @@ -2168,7 +2173,7 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", ] [[package]] @@ -2205,7 +2210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" dependencies = [ "libc", - "rand_chacha 0.3.0", + "rand_chacha 0.3.1", "rand_core 0.6.2", "rand_hc 0.3.0", ] @@ -2222,9 +2227,9 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core 0.6.2", @@ -2260,7 +2265,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", ] [[package]] @@ -2292,9 +2297,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ "autocfg", "crossbeam-deque", @@ -2304,13 +2309,13 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", "lazy_static", "num_cpus", ] @@ -2335,9 +2340,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick", "memchr", @@ -2346,12 +2351,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" -dependencies = [ - "byteorder", -] +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" @@ -2406,9 +2408,9 @@ dependencies = [ [[package]] name = "retain_mut" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" +checksum = "e9c17925a9027d298a4603d286befe3f9dc0e8ed02523141914eb628798d6e5b" [[package]] name = "ring" @@ -2460,6 +2462,15 @@ dependencies = [ "semver 0.11.0", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.3", +] + [[package]] name = "rustls" version = "0.19.1" @@ -2522,6 +2533,12 @@ dependencies = [ "semver-parser 0.10.2", ] +[[package]] +name = "semver" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f3aac57ee7f3272d8395c6e4f502f434f0e289fcd62876f70daa008c20dcabe" + [[package]] name = "semver-parser" version = "0.7.0" @@ -2629,22 +2646,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.125" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.125" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -2695,13 +2712,13 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.4" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" +checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", - "cpuid-bool", + "cpufeatures", "digest 0.9.0", "opaque-debug 0.3.0", ] @@ -2714,22 +2731,22 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", - "cpuid-bool", + "cpufeatures", "digest 0.9.0", "opaque-debug 0.3.0", ] [[package]] name = "signal-hook-registry" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ "libc", ] @@ -2819,11 +2836,11 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", "serde", "serde_derive", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -2833,13 +2850,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", "serde", "serde_derive", "serde_json", "sha1", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -2873,9 +2890,9 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -2891,11 +2908,11 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.72" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" +checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", "unicode-xid 0.2.2", ] @@ -2915,17 +2932,17 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", "unicode-xid 0.2.2", ] [[package]] name = "tar" -version = "0.4.33" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0bcfbd6a598361fda270d82469fff3d65089dc33e175c9a131f7b4cd395f228" +checksum = "7d779dc6aeff029314570f666ec83f19df7280bb36ef338442cfa8c604021b80" dependencies = [ "filetime", "libc", @@ -2976,22 +2993,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -3007,9 +3024,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.26" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a8cbfbf47955132d0202d1662f49b2423ae35862aee471f3ba4b133358f372" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" dependencies = [ "const_fn", "libc", @@ -3032,15 +3049,15 @@ dependencies = [ [[package]] name = "time-macros-impl" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", "standback", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -3070,9 +3087,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" +checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975" dependencies = [ "autocfg", "bytes 1.0.1", @@ -3090,13 +3107,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", ] [[package]] @@ -3112,9 +3129,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940a12c99365c31ea8dd9ba04ec1be183ffe4920102bb7122c2f515437601e8e" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" dependencies = [ "bytes 1.0.1", "futures-core", @@ -3212,9 +3229,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" dependencies = [ "tinyvec", ] @@ -3251,9 +3268,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", "idna", @@ -3264,9 +3281,9 @@ dependencies = [ [[package]] name = "urlencoding" -version = "1.1.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" +checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb" [[package]] name = "utf8-width" @@ -3280,7 +3297,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", "serde", ] @@ -3298,7 +3315,7 @@ checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" dependencies = [ "bitflags", "chrono", - "rustc_version 0.3.3", + "rustc_version 0.4.0", ] [[package]] @@ -3342,9 +3359,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" +checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", "serde", @@ -3354,24 +3371,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" +checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" +checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3381,9 +3398,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" +checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" dependencies = [ "quote 1.0.9", "wasm-bindgen-macro-support", @@ -3391,28 +3408,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" +checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" dependencies = [ - "proc-macro2 1.0.26", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.72", + "syn 1.0.73", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" +checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" [[package]] name = "web-sys" -version = "0.3.50" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" dependencies = [ "js-sys", "wasm-bindgen", @@ -3511,16 +3528,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ - "proc-macro2 1.0.26", - "syn 1.0.72", + "proc-macro2 1.0.27", + "syn 1.0.73", "synstructure", ] [[package]] name = "zip" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c83dc9b784d252127720168abd71ea82bf8c3d96b17dc565b5e2a02854f2b27" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" dependencies = [ "byteorder", "bzip2", diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index c74cf11f5..eebbf6247 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -4,6 +4,8 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicBool; use std::sync::Arc; +use async_stream::stream; +use futures::StreamExt; use log::info; use oxidized_json_checker::JsonChecker; use tokio::fs; @@ -18,7 +20,7 @@ use crate::index_controller::{UpdateMeta, UpdateStatus}; pub struct UpdateActor { path: PathBuf, store: Arc, - inbox: mpsc::Receiver>, + inbox: Option>>, index_handle: I, must_exit: Arc, } @@ -45,7 +47,7 @@ where let store = UpdateStore::open(options, &path, index_handle.clone(), must_exit.clone())?; std::fs::create_dir_all(path.join("update_files"))?; - + let inbox = Some(inbox); Ok(Self { path, store, @@ -60,43 +62,59 @@ where info!("Started update actor."); - loop { - let msg = self.inbox.recv().await; + let mut inbox = self + .inbox + .take() + .expect("A receiver should be present by now."); - if self.must_exit.load(std::sync::atomic::Ordering::Relaxed) { - break; - } + let must_exit = self.must_exit.clone(); + let stream = stream! { + loop { + let msg = inbox.recv().await; - match msg { - Some(Update { - uuid, - meta, - data, - ret, - }) => { - let _ = ret.send(self.handle_update(uuid, meta, data).await); + if must_exit.load(std::sync::atomic::Ordering::Relaxed) { + break; } - Some(ListUpdates { uuid, ret }) => { - let _ = ret.send(self.handle_list_updates(uuid).await); + + match msg { + Some(msg) => yield msg, + None => break, } - Some(GetUpdate { uuid, ret, id }) => { - let _ = ret.send(self.handle_get_update(uuid, id).await); - } - Some(Delete { uuid, ret }) => { - let _ = ret.send(self.handle_delete(uuid).await); - } - Some(Snapshot { uuids, path, ret }) => { - let _ = ret.send(self.handle_snapshot(uuids, path).await); - } - Some(Dump { uuids, path, ret }) => { - let _ = ret.send(self.handle_dump(uuids, path).await); - } - Some(GetInfo { ret }) => { - let _ = ret.send(self.handle_get_info().await); - } - None => break, } - } + }; + + stream + .for_each_concurrent(Some(10), |msg| async { + match msg { + Update { + uuid, + meta, + data, + ret, + } => { + let _ = ret.send(self.handle_update(uuid, meta, data).await); + } + ListUpdates { uuid, ret } => { + let _ = ret.send(self.handle_list_updates(uuid).await); + } + GetUpdate { uuid, ret, id } => { + let _ = ret.send(self.handle_get_update(uuid, id).await); + } + Delete { uuid, ret } => { + let _ = ret.send(self.handle_delete(uuid).await); + } + Snapshot { uuids, path, ret } => { + let _ = ret.send(self.handle_snapshot(uuids, path).await); + } + GetInfo { ret } => { + let _ = ret.send(self.handle_get_info().await); + } + Dump { uuids, path, ret } => { + let _ = ret.send(self.handle_dump(uuids, path).await); + } + } + }) + .await; } async fn handle_update( From 11c81ab4cb4998afc8511e2d007281f833939359 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Mon, 3 May 2021 14:21:09 +0200 Subject: [PATCH 382/522] fix tests --- meilisearch-http/src/routes/mod.rs | 16 ++++---------- .../tests/documents/add_documents.rs | 21 ++++++++++--------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 71a62c457..e1db8147d 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -4,7 +4,7 @@ use actix_web::{get, HttpResponse}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::index::Settings; +use crate::index::{Settings, Unchecked}; use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus}; pub mod document; @@ -34,7 +34,7 @@ pub enum UpdateType { #[serde(skip_serializing_if = "Option::is_none")] number: Option }, - Settings { settings: Settings }, + Settings { settings: Settings }, } impl From<&UpdateStatus> for UpdateType { @@ -60,20 +60,12 @@ impl From<&UpdateStatus> for UpdateType { } } UpdateMeta::ClearDocuments => UpdateType::ClearAll, - UpdateMeta::DeleteDocuments => { - let number = match other { - UpdateStatus::Processed(processed) => match processed.success { - UpdateResult::DocumentDeletion { deleted } => Some(deleted as usize), - _ => None, - }, - _ => None, - }; - UpdateType::DocumentsDeletion { number } + UpdateMeta::DeleteDocuments { ids } => { + UpdateType::DocumentsDeletion { number: Some(ids.len()) } } UpdateMeta::Settings(settings) => UpdateType::Settings { settings: settings.clone(), }, - _ => unreachable!(), } } } diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 1ec84e046..5d2a91674 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -33,16 +33,14 @@ async fn add_documents_no_index_creation() { assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["updateId"], 0); - assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + assert_eq!(response["type"]["name"], "DocumentsAddition"); + assert_eq!(response["type"]["number"], 1); let processed_at = DateTime::parse_from_rfc3339(response["processedAt"].as_str().unwrap()).unwrap(); let enqueued_at = DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap(); - let started_processing_at = - DateTime::parse_from_rfc3339(response["startedProcessingAt"].as_str().unwrap()).unwrap(); - assert!(processed_at > started_processing_at); - assert!(started_processing_at > enqueued_at); + assert!(processed_at > enqueued_at); // index was created, and primary key was infered. let (response, code) = index.get().await; @@ -86,7 +84,8 @@ async fn document_addition_with_primary_key() { assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["updateId"], 0); - assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + assert_eq!(response["type"]["name"], "DocumentsAddition"); + assert_eq!(response["type"]["number"], 1); let (response, code) = index.get().await; assert_eq!(code, 200); @@ -113,7 +112,8 @@ async fn document_update_with_primary_key() { assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["updateId"], 0); - assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); + assert_eq!(response["type"]["name"], "DocumentsPartial"); + assert_eq!(response["type"]["number"], 1); let (response, code) = index.get().await; assert_eq!(code, 200); @@ -282,7 +282,8 @@ async fn add_larger_dataset() { let (response, code) = index.get_update(update_id).await; assert_eq!(code, 200); assert_eq!(response["status"], "processed"); - assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); + assert_eq!(response["type"]["name"], "DocumentsAddition"); + assert_eq!(response["type"]["number"], 77); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { limit: Some(1000), @@ -302,8 +303,8 @@ async fn update_larger_dataset() { index.wait_update_id(0).await; let (response, code) = index.get_update(0).await; assert_eq!(code, 200); - assert_eq!(response["status"], "processed"); - assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); + assert_eq!(response["type"]["name"], "DocumentsPartial"); + assert_eq!(response["type"]["number"], 77); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { limit: Some(1000), From aa04124bfcebf91e6bb6d18c3baaf2d92a9d36d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 3 Jun 2021 19:36:25 +0200 Subject: [PATCH 383/522] Add changes according to milli update --- Cargo.lock | 4 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/index/mod.rs | 5 +- meilisearch-http/src/index/search.rs | 46 +++++++++++++++++-- meilisearch-http/src/index/updates.rs | 12 ++--- .../index_controller/dump_actor/loaders/v1.rs | 2 +- meilisearch-http/src/routes/settings/mod.rs | 2 +- 7 files changed, 55 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8546e3bc..415895d95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1658,8 +1658,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.2.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.2.1#25f75d4d03732131e6edcf20f4d126210b159d43" +version = "0.3.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.3.0#a32236c80cb1334cf249d820e8614ee36ace166e" dependencies = [ "anyhow", "bstr", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 77ca80438..997474e81 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.3.0" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 790ac58f0..4eddbfb8b 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -67,7 +67,6 @@ impl Index { let faceted_attributes = self .faceted_fields(&txn)? .into_iter() - .map(|(k, v)| (k, v.to_string())) .collect(); let criteria = self @@ -83,7 +82,7 @@ impl Index { }) .transpose()? .unwrap_or_else(BTreeSet::new); - let distinct_attribute = self.distinct_attribute(&txn)?.map(String::from); + let distinct_field = self.distinct_field(&txn)?.map(String::from); Ok(Settings { displayed_attributes: Some(displayed_attributes), @@ -91,7 +90,7 @@ impl Index { attributes_for_faceting: Some(Some(faceted_attributes)), ranking_rules: Some(Some(criteria)), stop_words: Some(Some(stop_words)), - distinct_attribute: Some(distinct_attribute), + distinct_attribute: Some(distinct_field), _kind: PhantomData, }) } diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index db0700d89..1c9f9899a 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -8,7 +8,7 @@ use heed::RoTxn; use indexmap::IndexMap; use itertools::Itertools; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{facet::FacetValue, FacetCondition, FieldId, FieldsIdsMap, MatchingWords}; +use milli::{FilterCondition, FieldId, FieldsIdsMap, MatchingWords}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -57,7 +57,7 @@ pub struct SearchResult { pub offset: usize, pub processing_time_ms: u128, #[serde(skip_serializing_if = "Option::is_none")] - pub facet_distributions: Option>>, + pub facet_distributions: Option>>, } impl Index { @@ -74,9 +74,15 @@ impl Index { search.limit(query.limit); search.offset(query.offset.unwrap_or_default()); +<<<<<<< HEAD if let Some(ref filter) = query.filter { if let Some(facets) = parse_facets(filter, self, &rtxn)? { search.facet_condition(facets); +======= + if let Some(ref facets) = query.facet_filters { + if let Some(facets) = parse_facets(facets, self, &rtxn)? { + search.filter(facets); +>>>>>>> 562cc32 (Add changes according to milli update) } } @@ -272,10 +278,42 @@ impl Matcher for HashSet { impl Matcher for MatchingWords { fn matches(&self, w: &str) -> bool { - self.matches(w) + self.matching_bytes(w).is_some() } } +<<<<<<< HEAD +======= +fn parse_facets_array( + txn: &RoTxn, + index: &Index, + arr: &[Value], +) -> anyhow::Result> { + let mut ands = Vec::new(); + for value in arr { + match value { + Value::String(s) => ands.push(Either::Right(s.clone())), + Value::Array(arr) => { + let mut ors = Vec::new(); + for value in arr { + match value { + Value::String(s) => ors.push(s.clone()), + v => bail!("Invalid facet expression, expected String, found: {:?}", v), + } + } + ands.push(Either::Left(ors)); + } + v => bail!( + "Invalid facet expression, expected String or [String], found: {:?}", + v + ), + } + } + + FilterCondition::from_array(txn, &index.0, ands) +} + +>>>>>>> 562cc32 (Add changes according to milli update) struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, marks: (String, String), @@ -335,7 +373,7 @@ fn parse_facets( facets: &Value, index: &Index, txn: &RoTxn, -) -> anyhow::Result> { +) -> anyhow::Result> { match facets { Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), Value::Array(arr) => parse_facets_array(txn, index, arr), diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 9ed4fe49e..193a52672 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashSet}; use std::io; use std::marker::PhantomData; use std::num::NonZeroUsize; @@ -51,7 +51,7 @@ pub struct Settings { deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none" )] - pub attributes_for_faceting: Option>>, + pub attributes_for_faceting: Option>>, #[serde( default, @@ -253,8 +253,8 @@ impl Index { } if let Some(ref facet_types) = settings.attributes_for_faceting { - let facet_types = facet_types.clone().unwrap_or_else(HashMap::new); - builder.set_faceted_fields(facet_types); + let facet_types = facet_types.clone().unwrap_or_else(HashSet::new); + builder.set_filterable_fields(facet_types); } if let Some(ref criteria) = settings.ranking_rules { @@ -273,8 +273,8 @@ impl Index { if let Some(ref distinct_attribute) = settings.distinct_attribute { match distinct_attribute { - Some(attr) => builder.set_distinct_attribute(attr.clone()), - None => builder.reset_distinct_attribute(), + Some(attr) => builder.set_distinct_field(attr.clone()), + None => builder.reset_distinct_field(), } } diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index decd67f87..a1f14e8a1 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -145,7 +145,7 @@ impl From for index_controller::Settings { // representing the name of the faceted field + the type of the field. Since the type // was not known in the V1 of the dump we are just going to assume everything is a // String - attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().map(|key| (key, String::from("string"))).collect())), + attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().collect())), // we need to convert the old `Vec` into a `BTreeSet` ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { match criterion.as_str() { diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 8ede56046..008ea1576 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -75,7 +75,7 @@ macro_rules! make_setting_route { make_setting_route!( "/indexes/{index_uid}/settings/attributes-for-faceting", - std::collections::HashMap, + std::collections::HashSet, attributes_for_faceting ); From ca1bb7dc1ccc810ab8e670e6ad782ed5f206c0b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 3 Jun 2021 19:45:10 +0200 Subject: [PATCH 384/522] Fix tests --- meilisearch-http/tests/settings/get_settings.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index ab688076d..3ce77b2a6 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -19,7 +19,7 @@ async fn get_settings() { assert_eq!(settings.keys().len(), 6); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); - assert_eq!(settings["attributesForFaceting"], json!({})); + assert_eq!(settings["attributesForFaceting"], json!([])); assert_eq!(settings["distinctAttribute"], json!(null)); assert_eq!( settings["rankingRules"], @@ -73,7 +73,7 @@ async fn reset_all_settings() { let server = Server::new().await; let index = server.index("test"); index - .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"], "stopWords": ["the"], "attributesForFaceting": { "toto": "string" } })) + .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"], "stopWords": ["the"], "attributesForFaceting": ["toto"] })) .await; index.wait_update_id(0).await; let (response, code) = index.settings().await; @@ -81,7 +81,7 @@ async fn reset_all_settings() { assert_eq!(response["displayedAttributes"], json!(["foo"])); assert_eq!(response["searchableAttributes"], json!(["bar"])); assert_eq!(response["stopWords"], json!(["the"])); - assert_eq!(response["attributesForFaceting"], json!({"toto": "string"})); + assert_eq!(response["attributesForFaceting"], json!(["toto"])); index.delete_settings().await; index.wait_update_id(1).await; @@ -91,7 +91,7 @@ async fn reset_all_settings() { assert_eq!(response["displayedAttributes"], json!(["*"])); assert_eq!(response["searchableAttributes"], json!(["*"])); assert_eq!(response["stopWords"], json!([])); - assert_eq!(response["attributesForFaceting"], json!({})); + assert_eq!(response["attributesForFaceting"], json!([])); } #[actix_rt::test] From 7009906d557abc79db1197996c58eb2d48d4e34b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 3 Jun 2021 20:08:31 +0200 Subject: [PATCH 385/522] Update reset-all-settings test --- .../tests/settings/get_settings.rs | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 3ce77b2a6..fc5cacca0 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -72,19 +72,33 @@ async fn delete_settings_unexisting_index() { async fn reset_all_settings() { let server = Server::new().await; let index = server.index("test"); - index - .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"], "stopWords": ["the"], "attributesForFaceting": ["toto"] })) - .await; + + let documents = json!([ + { + "id": 1, + "name": "curqui", + "age": 99 + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + assert_eq!(response["updateId"], 0); index.wait_update_id(0).await; + + index + .update_settings(json!({"displayedAttributes": ["name", "age"], "searchableAttributes": ["name"], "stopWords": ["the"], "attributesForFaceting": ["age"] })) + .await; + index.wait_update_id(1).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"], json!(["foo"])); - assert_eq!(response["searchableAttributes"], json!(["bar"])); + assert_eq!(response["displayedAttributes"], json!(["name", "age"])); + assert_eq!(response["searchableAttributes"], json!(["name"])); assert_eq!(response["stopWords"], json!(["the"])); - assert_eq!(response["attributesForFaceting"], json!(["toto"])); + assert_eq!(response["attributesForFaceting"], json!(["age"])); index.delete_settings().await; - index.wait_update_id(1).await; + index.wait_update_id(2).await; let (response, code) = index.settings().await; assert_eq!(code, 200); @@ -92,6 +106,10 @@ async fn reset_all_settings() { assert_eq!(response["searchableAttributes"], json!(["*"])); assert_eq!(response["stopWords"], json!([])); assert_eq!(response["attributesForFaceting"], json!([])); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert!(response.as_object().unwrap().get("age").is_some()); } #[actix_rt::test] From 88bf867a3eceba3f87f2ac40f8c70ea4961736f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 3 Jun 2021 23:47:16 +0200 Subject: [PATCH 386/522] Rename attributes for faceting into filterable attributes --- meilisearch-http/src/index/mod.rs | 2 +- meilisearch-http/src/index/updates.rs | 16 ++++++++-------- .../index_controller/dump_actor/loaders/v1.rs | 4 ++-- meilisearch-http/src/routes/settings/mod.rs | 6 +++--- .../tests/assets/dumps/v1/test/settings.json | 2 +- .../tests/assets/dumps/v1/test/updates.jsonl | 4 ++-- meilisearch-http/tests/settings/get_settings.rs | 10 +++++----- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 4eddbfb8b..782c40d37 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -87,7 +87,7 @@ impl Index { Ok(Settings { displayed_attributes: Some(displayed_attributes), searchable_attributes: Some(searchable_attributes), - attributes_for_faceting: Some(Some(faceted_attributes)), + filterable_attributes: Some(Some(faceted_attributes)), ranking_rules: Some(Some(criteria)), stop_words: Some(Some(stop_words)), distinct_attribute: Some(distinct_field), diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 193a52672..fa26f6bee 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -51,7 +51,7 @@ pub struct Settings { deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none" )] - pub attributes_for_faceting: Option>>, + pub filterable_attributes: Option>>, #[serde( default, @@ -81,7 +81,7 @@ impl Settings { Settings { displayed_attributes: Some(None), searchable_attributes: Some(None), - attributes_for_faceting: Some(None), + filterable_attributes: Some(None), ranking_rules: Some(None), stop_words: Some(None), distinct_attribute: Some(None), @@ -93,7 +93,7 @@ impl Settings { let Self { displayed_attributes, searchable_attributes, - attributes_for_faceting, + filterable_attributes, ranking_rules, stop_words, distinct_attribute, @@ -103,7 +103,7 @@ impl Settings { Settings { displayed_attributes, searchable_attributes, - attributes_for_faceting, + filterable_attributes, ranking_rules, stop_words, distinct_attribute, @@ -139,7 +139,7 @@ impl Settings { Settings { displayed_attributes, searchable_attributes, - attributes_for_faceting: self.attributes_for_faceting, + filterable_attributes: self.filterable_attributes, ranking_rules: self.ranking_rules, stop_words: self.stop_words, distinct_attribute: self.distinct_attribute, @@ -252,7 +252,7 @@ impl Index { } } - if let Some(ref facet_types) = settings.attributes_for_faceting { + if let Some(ref facet_types) = settings.filterable_attributes { let facet_types = facet_types.clone().unwrap_or_else(HashSet::new); builder.set_filterable_fields(facet_types); } @@ -329,7 +329,7 @@ mod test { let settings = Settings { displayed_attributes: Some(Some(vec![String::from("hello")])), searchable_attributes: Some(Some(vec![String::from("hello")])), - attributes_for_faceting: None, + filterable_attributes: None, ranking_rules: None, stop_words: None, distinct_attribute: None, @@ -348,7 +348,7 @@ mod test { let settings = Settings { displayed_attributes: Some(Some(vec![String::from("*")])), searchable_attributes: Some(Some(vec![String::from("hello"), String::from("*")])), - attributes_for_faceting: None, + filterable_attributes: None, ranking_rules: None, stop_words: None, distinct_attribute: None, diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index a1f14e8a1..247b81a95 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -73,7 +73,7 @@ struct Settings { #[serde(default, deserialize_with = "deserialize_some")] pub synonyms: Option>>>, #[serde(default, deserialize_with = "deserialize_some")] - pub attributes_for_faceting: Option>>, + pub filterable_attributes: Option>>, } fn load_index( @@ -145,7 +145,7 @@ impl From for index_controller::Settings { // representing the name of the faceted field + the type of the field. Since the type // was not known in the V1 of the dump we are just going to assume everything is a // String - attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().collect())), + filterable_attributes: settings.filterable_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), // we need to convert the old `Vec` into a `BTreeSet` ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { match criterion.as_str() { diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 008ea1576..ca8dd03ce 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -74,9 +74,9 @@ macro_rules! make_setting_route { } make_setting_route!( - "/indexes/{index_uid}/settings/attributes-for-faceting", + "/indexes/{index_uid}/settings/filterable-attributes", std::collections::HashSet, - attributes_for_faceting + filterable_attributes ); make_setting_route!( @@ -126,7 +126,7 @@ macro_rules! create_services { } create_services!( - attributes_for_faceting, + filterable_attributes, displayed_attributes, searchable_attributes, distinct_attribute, diff --git a/meilisearch-http/tests/assets/dumps/v1/test/settings.json b/meilisearch-http/tests/assets/dumps/v1/test/settings.json index 918cfab53..c000bc7f6 100644 --- a/meilisearch-http/tests/assets/dumps/v1/test/settings.json +++ b/meilisearch-http/tests/assets/dumps/v1/test/settings.json @@ -50,7 +50,7 @@ "wolverine": ["xmen", "logan"], "logan": ["wolverine", "xmen"] }, - "attributesForFaceting": [ + "filterableAttributes": [ "gender", "color", "tags", diff --git a/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl index 0dcffdce0..9eb50e43e 100644 --- a/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl +++ b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl @@ -1,2 +1,2 @@ -{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}} -{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} \ No newline at end of file +{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","filterable_attributes":"Nothing"}}} +{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index fc5cacca0..4fd9fa724 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -19,7 +19,7 @@ async fn get_settings() { assert_eq!(settings.keys().len(), 6); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); - assert_eq!(settings["attributesForFaceting"], json!([])); + assert_eq!(settings["filterableAttributes"], json!([])); assert_eq!(settings["distinctAttribute"], json!(null)); assert_eq!( settings["rankingRules"], @@ -87,7 +87,7 @@ async fn reset_all_settings() { index.wait_update_id(0).await; index - .update_settings(json!({"displayedAttributes": ["name", "age"], "searchableAttributes": ["name"], "stopWords": ["the"], "attributesForFaceting": ["age"] })) + .update_settings(json!({"displayedAttributes": ["name", "age"], "searchableAttributes": ["name"], "stopWords": ["the"], "filterableAttributes": ["age"] })) .await; index.wait_update_id(1).await; let (response, code) = index.settings().await; @@ -95,7 +95,7 @@ async fn reset_all_settings() { assert_eq!(response["displayedAttributes"], json!(["name", "age"])); assert_eq!(response["searchableAttributes"], json!(["name"])); assert_eq!(response["stopWords"], json!(["the"])); - assert_eq!(response["attributesForFaceting"], json!(["age"])); + assert_eq!(response["filterableAttributes"], json!(["age"])); index.delete_settings().await; index.wait_update_id(2).await; @@ -105,7 +105,7 @@ async fn reset_all_settings() { assert_eq!(response["displayedAttributes"], json!(["*"])); assert_eq!(response["searchableAttributes"], json!(["*"])); assert_eq!(response["stopWords"], json!([])); - assert_eq!(response["attributesForFaceting"], json!([])); + assert_eq!(response["filterableAttributes"], json!([])); let (response, code) = index.get_document(1, None).await; assert_eq!(code, 200); @@ -181,7 +181,7 @@ macro_rules! test_setting_routes { } test_setting_routes!( - attributes_for_faceting, + filterable_attributes, displayed_attributes, searchable_attributes, stop_words From 9996c59183a492f402f0a156682bedbf933ca2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 9 Jun 2021 18:05:17 +0200 Subject: [PATCH 387/522] Update with milli 0.3.1 --- Cargo.lock | 5 ++--- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 415895d95..75d05631a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1658,14 +1658,13 @@ dependencies = [ [[package]] name = "milli" -version = "0.3.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.3.0#a32236c80cb1334cf249d820e8614ee36ace166e" +version = "0.3.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.3.1#bc020317935da4ea08061b3d4518cbbd40184856" dependencies = [ "anyhow", "bstr", "byteorder", "chrono", - "crossbeam-channel", "csv", "either", "flate2", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 997474e81..64efe4c62 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.3.0" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.3.1" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" From d4be4d80db59f8b59598491cac2742962d8b7716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 14 Jun 2021 13:27:18 +0200 Subject: [PATCH 388/522] Fix after rebase --- meilisearch-http/src/index/search.rs | 44 ++-------------------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 1c9f9899a..50d163898 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -74,15 +74,9 @@ impl Index { search.limit(query.limit); search.offset(query.offset.unwrap_or_default()); -<<<<<<< HEAD if let Some(ref filter) = query.filter { if let Some(facets) = parse_facets(filter, self, &rtxn)? { - search.facet_condition(facets); -======= - if let Some(ref facets) = query.facet_filters { - if let Some(facets) = parse_facets(facets, self, &rtxn)? { search.filter(facets); ->>>>>>> 562cc32 (Add changes according to milli update) } } @@ -282,38 +276,6 @@ impl Matcher for MatchingWords { } } -<<<<<<< HEAD -======= -fn parse_facets_array( - txn: &RoTxn, - index: &Index, - arr: &[Value], -) -> anyhow::Result> { - let mut ands = Vec::new(); - for value in arr { - match value { - Value::String(s) => ands.push(Either::Right(s.clone())), - Value::Array(arr) => { - let mut ors = Vec::new(); - for value in arr { - match value { - Value::String(s) => ors.push(s.clone()), - v => bail!("Invalid facet expression, expected String, found: {:?}", v), - } - } - ands.push(Either::Left(ors)); - } - v => bail!( - "Invalid facet expression, expected String or [String], found: {:?}", - v - ), - } - } - - FilterCondition::from_array(txn, &index.0, ands) -} - ->>>>>>> 562cc32 (Add changes according to milli update) struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, marks: (String, String), @@ -375,7 +337,7 @@ fn parse_facets( txn: &RoTxn, ) -> anyhow::Result> { match facets { - Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), + Value::String(expr) => Ok(Some(FilterCondition::from_str(txn, index, expr)?)), Value::Array(arr) => parse_facets_array(txn, index, arr), v => bail!("Invalid facet expression, expected Array, found: {:?}", v), } @@ -385,7 +347,7 @@ fn parse_facets_array( txn: &RoTxn, index: &Index, arr: &[Value], -) -> anyhow::Result> { +) -> anyhow::Result> { let mut ands = Vec::new(); for value in arr { match value { @@ -407,7 +369,7 @@ fn parse_facets_array( } } - FacetCondition::from_array(txn, &index.0, ands) + FilterCondition::from_array(txn, &index.0, ands) } #[cfg(test)] From 18d4d6097a536a7d10856784d354913dc068fb60 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 3 Jun 2021 14:19:56 +0200 Subject: [PATCH 389/522] implements the synonyms in transplant --- meilisearch-http/src/index/mod.rs | 12 +++++ meilisearch-http/src/index/updates.rs | 21 ++++++++- .../index_controller/dump_actor/loaders/v1.rs | 5 +- meilisearch-http/src/lib.rs | 1 - meilisearch-http/src/routes/mod.rs | 1 - meilisearch-http/src/routes/settings/mod.rs | 7 +++ meilisearch-http/src/routes/synonym.rs | 47 ------------------- .../tests/settings/get_settings.rs | 12 +++-- 8 files changed, 50 insertions(+), 56 deletions(-) delete mode 100644 meilisearch-http/src/routes/synonym.rs diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 782c40d37..cb942a7b8 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -84,6 +84,17 @@ impl Index { .unwrap_or_else(BTreeSet::new); let distinct_field = self.distinct_field(&txn)?.map(String::from); + let synonyms = self + .synonyms(&txn)? + .iter() + .map(|(key, values)| { + ( + key.join(" "), + values.iter().map(|value| value.join(" ")).collect(), + ) + }) + .collect(); + Ok(Settings { displayed_attributes: Some(displayed_attributes), searchable_attributes: Some(searchable_attributes), @@ -91,6 +102,7 @@ impl Index { ranking_rules: Some(Some(criteria)), stop_words: Some(Some(stop_words)), distinct_attribute: Some(distinct_field), + synonyms: Some(Some(synonyms)), _kind: PhantomData, }) } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index fa26f6bee..3e31cd75c 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashSet}; +use std::collections::{BTreeSet, BTreeMap, HashSet}; use std::io; use std::marker::PhantomData; use std::num::NonZeroUsize; @@ -70,6 +70,12 @@ pub struct Settings { deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none" )] + pub synonyms: Option>>>, + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] pub distinct_attribute: Option>, #[serde(skip)] @@ -84,6 +90,7 @@ impl Settings { filterable_attributes: Some(None), ranking_rules: Some(None), stop_words: Some(None), + synonyms: Some(None), distinct_attribute: Some(None), _kind: PhantomData, } @@ -96,6 +103,7 @@ impl Settings { filterable_attributes, ranking_rules, stop_words, + synonyms, distinct_attribute, .. } = self; @@ -106,6 +114,7 @@ impl Settings { filterable_attributes, ranking_rules, stop_words, + synonyms, distinct_attribute, _kind: PhantomData, } @@ -142,6 +151,7 @@ impl Settings { filterable_attributes: self.filterable_attributes, ranking_rules: self.ranking_rules, stop_words: self.stop_words, + synonyms: self.synonyms, distinct_attribute: self.distinct_attribute, _kind: PhantomData, } @@ -271,6 +281,13 @@ impl Index { } } + if let Some(ref synonyms) = settings.synonyms { + match synonyms { + Some(synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()), + _ => builder.reset_synonyms(), + } + } + if let Some(ref distinct_attribute) = settings.distinct_attribute { match distinct_attribute { Some(attr) => builder.set_distinct_field(attr.clone()), @@ -332,6 +349,7 @@ mod test { filterable_attributes: None, ranking_rules: None, stop_words: None, + synonyms: None, distinct_attribute: None, _kind: PhantomData::, }; @@ -351,6 +369,7 @@ mod test { filterable_attributes: None, ranking_rules: None, stop_words: None, + synonyms: None, distinct_attribute: None, _kind: PhantomData::, }; diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 247b81a95..a7f1aa8d1 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -133,9 +133,6 @@ fn load_index( /// we need to **always** be able to convert the old settings to the settings currently being used impl From for index_controller::Settings { fn from(settings: Settings) -> Self { - if settings.synonyms.flatten().is_some() { - error!("`synonyms` are not yet implemented and thus will be ignored"); - } Self { distinct_attribute: settings.distinct_attribute, // we need to convert the old `Vec` into a `BTreeSet` @@ -167,6 +164,8 @@ impl From for index_controller::Settings { }).collect())), // we need to convert the old `Vec` into a `BTreeSet` stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())), + // we need to convert the old `Vec` into a `BTreeMap` + synonyms: settings.synonyms.map(|o| o.map(|vec| vec.into_iter().collect())), _kind: PhantomData, } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 26b6a784c..fb80aa3d6 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -43,7 +43,6 @@ macro_rules! create_app { .configure(index::services) .configure(search::services) .configure(settings::services) - .configure(synonym::services) .configure(health::services) .configure(stats::services) .configure(key::services) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index e1db8147d..948e263cc 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -15,7 +15,6 @@ pub mod key; pub mod search; pub mod settings; pub mod stats; -pub mod synonym; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "name")] diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index ca8dd03ce..45d49132c 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -97,6 +97,12 @@ make_setting_route!( stop_words ); +make_setting_route!( + "/indexes/{index_uid}/settings/synonyms", + std::collections::BTreeMap>, + synonyms +); + make_setting_route!( "/indexes/{index_uid}/settings/distinct-attribute", String, @@ -131,6 +137,7 @@ create_services!( searchable_attributes, distinct_attribute, stop_words, + synonyms, ranking_rules ); diff --git a/meilisearch-http/src/routes/synonym.rs b/meilisearch-http/src/routes/synonym.rs deleted file mode 100644 index 4106e3754..000000000 --- a/meilisearch-http/src/routes/synonym.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::collections::BTreeMap; - -use actix_web::{delete, get, post}; -use actix_web::{web, HttpResponse}; - -use crate::error::ResponseError; -use crate::helpers::Authentication; -use crate::routes::IndexParam; -use crate::Data; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get).service(update).service(delete); -} - -#[get( - "/indexes/{index_uid}/settings/synonyms", - wrap = "Authentication::Private" -)] -async fn get( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} - -#[post( - "/indexes/{index_uid}/settings/synonyms", - wrap = "Authentication::Private" -)] -async fn update( - _data: web::Data, - _path: web::Path, - _body: web::Json>>, -) -> Result { - todo!() -} - -#[delete( - "/indexes/{index_uid}/settings/synonyms", - wrap = "Authentication::Private" -)] -async fn delete( - _data: web::Data, - _path: web::Path, -) -> Result { - todo!() -} diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 4fd9fa724..a4ee6a4d3 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -16,7 +16,7 @@ async fn get_settings() { let (response, code) = index.settings().await; assert_eq!(code, 200); let settings = response.as_object().unwrap(); - assert_eq!(settings.keys().len(), 6); + assert_eq!(settings.keys().len(), 7); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["filterableAttributes"], json!([])); @@ -87,7 +87,7 @@ async fn reset_all_settings() { index.wait_update_id(0).await; index - .update_settings(json!({"displayedAttributes": ["name", "age"], "searchableAttributes": ["name"], "stopWords": ["the"], "filterableAttributes": ["age"] })) + .update_settings(json!({"displayedAttributes": ["name", "age"], "searchableAttributes": ["name"], "stopWords": ["the"], "filterableAttributes": ["age"], "synonyms": {"puppy": ["dog", "doggo", "potat"] }})) .await; index.wait_update_id(1).await; let (response, code) = index.settings().await; @@ -95,6 +95,10 @@ async fn reset_all_settings() { assert_eq!(response["displayedAttributes"], json!(["name", "age"])); assert_eq!(response["searchableAttributes"], json!(["name"])); assert_eq!(response["stopWords"], json!(["the"])); + assert_eq!( + response["synonyms"], + json!({"puppy": ["dog", "doggo", "potat"] }) + ); assert_eq!(response["filterableAttributes"], json!(["age"])); index.delete_settings().await; @@ -110,6 +114,7 @@ async fn reset_all_settings() { let (response, code) = index.get_document(1, None).await; assert_eq!(code, 200); assert!(response.as_object().unwrap().get("age").is_some()); + assert_eq!(response["synonyms"], json!({})); } #[actix_rt::test] @@ -184,5 +189,6 @@ test_setting_routes!( filterable_attributes, displayed_attributes, searchable_attributes, - stop_words + stop_words, + synonyms ); From f068d7f978b19eb572e66799f9198928b5017c29 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 14 Jun 2021 10:34:16 +0200 Subject: [PATCH 390/522] makes clippy happy --- meilisearch-http/src/index_controller/updates.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index c2f7db30e..abeec413e 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -14,6 +14,7 @@ pub enum UpdateResult { Other, } +#[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] pub enum UpdateMeta { From 7cb2dcbdf88ec678008fb1d6983d2432e3e81aaa Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 14 Jun 2021 10:38:56 +0200 Subject: [PATCH 391/522] add a comment --- meilisearch-http/src/index/mod.rs | 2 ++ meilisearch-http/src/index/updates.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index cb942a7b8..56958760a 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -84,6 +84,8 @@ impl Index { .unwrap_or_else(BTreeSet::new); let distinct_field = self.distinct_field(&txn)?.map(String::from); + // in milli each word in the synonyms map were split on their separator. Since we lost + // this information we are going to put space between words. let synonyms = self .synonyms(&txn)? .iter() diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 3e31cd75c..fc475aa4d 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -277,14 +277,14 @@ impl Index { if let Some(ref stop_words) = settings.stop_words { match stop_words { Some(stop_words) => builder.set_stop_words(stop_words.clone()), - _ => builder.reset_stop_words(), + None => builder.reset_stop_words(), } } if let Some(ref synonyms) = settings.synonyms { match synonyms { Some(synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()), - _ => builder.reset_synonyms(), + None => builder.reset_synonyms(), } } From a780cff8fd19ad92383ce9b820c384d5fcbe6bdc Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 14 Jun 2021 14:35:40 +0200 Subject: [PATCH 392/522] fix clippy warning --- meilisearch-http/src/routes/mod.rs | 1 + meilisearch-http/tests/settings/get_settings.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 948e263cc..beddf7ee3 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -17,6 +17,7 @@ pub mod settings; pub mod stats; #[derive(Debug, Clone, Serialize, Deserialize)] +#[allow(clippy::clippy::large_enum_variant)] #[serde(tag = "name")] pub enum UpdateType { ClearAll, diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index a4ee6a4d3..a03d907a7 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -110,11 +110,11 @@ async fn reset_all_settings() { assert_eq!(response["searchableAttributes"], json!(["*"])); assert_eq!(response["stopWords"], json!([])); assert_eq!(response["filterableAttributes"], json!([])); + assert_eq!(response["synonyms"], json!({})); let (response, code) = index.get_document(1, None).await; assert_eq!(code, 200); assert!(response.as_object().unwrap().get("age").is_some()); - assert_eq!(response["synonyms"], json!({})); } #[actix_rt::test] From fe5a49403554feaa5e9aeafcc6433f68b7b2b79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 14 Jun 2021 17:55:04 +0200 Subject: [PATCH 393/522] Update alpha for the next release --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4bc1f6ec9..f10741b55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1546,7 +1546,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.21.0-alpha.4" +version = "0.21.0-alpha.5" dependencies = [ "actix-cors", "actix-http", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 64efe4c62..23ccbe42c 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "MeiliSearch HTTP server" edition = "2018" license = "MIT" name = "meilisearch-http" -version = "0.21.0-alpha.4" +version = "0.21.0-alpha.5" [[bin]] name = "meilisearch" From def1596eaf95c56a964399c4e5ae0875e0b155ee Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 15 Jun 2021 15:36:30 +0200 Subject: [PATCH 394/522] Integrate amplitude MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit And merge the sentry and amplitude usage under one “Enable analytics” flag --- Cargo.lock | 160 ++++++++++++++++++++++++ meilisearch-http/Cargo.toml | 6 +- meilisearch-http/src/analytics.rs | 55 ++++---- meilisearch-http/src/lib.rs | 3 + meilisearch-http/src/main.rs | 65 +++++----- meilisearch-http/src/option.rs | 16 +-- meilisearch-http/tests/common/server.rs | 5 +- 7 files changed, 221 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4bc1f6ec9..080a67a59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" version = "0.4.0" @@ -641,6 +643,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + [[package]] name = "cow-utils" version = "0.1.2" @@ -887,6 +905,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1296,6 +1329,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.0.1", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.2.3" @@ -1614,6 +1660,7 @@ dependencies = [ "uuid", "vergen", "walkdir", + "whoami", "zip", ] @@ -1775,6 +1822,24 @@ dependencies = [ "syn 1.0.73", ] +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.19.1" @@ -1864,6 +1929,39 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-float" version = "2.5.1" @@ -2384,11 +2482,13 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "lazy_static", "log", "mime", + "native-tls", "percent-encoding", "pin-project-lite", "rustls", @@ -2396,6 +2496,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", + "tokio-native-tls", "tokio-rustls", "url", "wasm-bindgen", @@ -2498,6 +2599,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -2514,6 +2625,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.9.0" @@ -3115,6 +3249,16 @@ dependencies = [ "syn 1.0.73", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.22.0" @@ -3300,6 +3444,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa" + [[package]] name = "vec_map" version = "0.8.2" @@ -3462,6 +3612,16 @@ dependencies = [ "hashbrown 0.7.2", ] +[[package]] +name = "whoami" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abacf325c958dfeaf1046931d37f2a901b6dfe0968ee965a29e94c6766b2af6" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 64efe4c62..5bb9cc6a4 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -74,6 +74,8 @@ uuid = { version = "0.8.2", features = ["serde"] } walkdir = "2.3.2" obkv = "0.1.1" pin-project = "1.0.7" +whoami = { version = "1.1.2", optional = true } +reqwest = { version = "0.11.3", features = ["json"], optional = true } [dependencies.sentry] default-features = false @@ -109,8 +111,8 @@ mini-dashboard = [ "tempfile", "zip", ] -telemetry = ["sentry"] -default = ["telemetry", "mini-dashboard"] +analytics = ["sentry", "whoami", "reqwest"] +default = ["analytics", "mini-dashboard"] [target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" diff --git a/meilisearch-http/src/analytics.rs b/meilisearch-http/src/analytics.rs index a01cfabe0..11347175b 100644 --- a/meilisearch-http/src/analytics.rs +++ b/meilisearch-http/src/analytics.rs @@ -1,14 +1,10 @@ use std::hash::{Hash, Hasher}; -use std::{error, thread}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use log::error; +use log::debug; use serde::Serialize; -use serde_qs as qs; use siphasher::sip::SipHasher; -use walkdir::WalkDir; -use crate::helpers::EnvSizer; use crate::Data; use crate::Opt; @@ -22,26 +18,21 @@ struct EventProperties { } impl EventProperties { - fn from(data: Data) -> Result> { - let mut index_list = Vec::new(); + async fn from(data: Data) -> anyhow::Result { + let stats = data.index_controller.get_all_stats().await?; - let reader = data.db.main_read_txn()?; - - for index_uid in data.db.indexes_uids() { - if let Some(index) = data.db.open_index(&index_uid) { - let number_of_documents = index.main.number_of_documents(&reader)?; - index_list.push(number_of_documents); - } - } - - let database_size = data.env.size(); - - let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp()); + let database_size = stats.database_size; + let last_update_timestamp = stats.last_update.map(|u| u.timestamp()); + let number_of_documents = stats + .indexes + .values() + .map(|index| index.number_of_documents) + .collect(); Ok(EventProperties { database_size, last_update_timestamp, - number_of_documents: index_list, + number_of_documents, }) } } @@ -68,10 +59,10 @@ struct Event<'a> { #[derive(Debug, Serialize)] struct AmplitudeRequest<'a> { api_key: &'a str, - event: &'a str, + events: Vec>, } -pub fn analytics_sender(data: Data, opt: Opt) { +pub async fn analytics_sender(data: Data, opt: Opt) { let username = whoami::username(); let hostname = whoami::hostname(); let platform = whoami::platform(); @@ -93,7 +84,7 @@ pub fn analytics_sender(data: Data, opt: Opt) { let time = n.as_secs(); let event_type = "runtime_tick"; let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day - let event_properties = EventProperties::from(data.clone()).ok(); + let event_properties = EventProperties::from(data.clone()).await.ok(); let app_version = env!("CARGO_PKG_VERSION").to_string(); let app_version = app_version.as_str(); let user_email = std::env::var("MEILI_USER_EMAIL").ok(); @@ -114,20 +105,22 @@ pub fn analytics_sender(data: Data, opt: Opt) { user_properties, event_properties, }; - let event = serde_json::to_string(&event).unwrap(); let request = AmplitudeRequest { api_key: AMPLITUDE_API_KEY, - event: &event, + events: vec![event], }; - let body = qs::to_string(&request).unwrap(); - let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body); - if !response.ok() { - let body = response.into_string().unwrap(); - error!("Unsuccessful call to Amplitude: {}", body); + let response = reqwest::Client::new() + .post("https://api2.amplitude.com/2/httpapi") + .timeout(Duration::from_secs(60)) // 1 minute max + .json(&request) + .send() + .await; + if let Err(e) = response { + debug!("Unsuccessful call to Amplitude: {}", e); } - thread::sleep(Duration::from_secs(3600)) // one hour + tokio::time::sleep(Duration::from_secs(3600)).await; } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 26b6a784c..8788d45d1 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -6,6 +6,9 @@ mod index_controller; pub mod option; pub mod routes; +#[cfg(feature = "analytics")] +pub mod analytics; + pub use self::data::Data; pub use option::Opt; diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index a29299673..120684dfc 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -5,12 +5,16 @@ use main_error::MainError; use meilisearch_http::{create_app, Data, Opt}; use structopt::StructOpt; -//mod analytics; +#[cfg(feature = "analytics")] +use meilisearch_http::analytics; #[cfg(target_os = "linux")] #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +#[cfg(all(not(debug_assertions), feature = "analytics"))] +const SENTRY_DSN: &str = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337"; + #[actix_web::main] async fn main() -> Result<(), MainError> { let opt = Opt::from_args(); @@ -26,16 +30,17 @@ async fn main() -> Result<(), MainError> { .into(), ); } - #[cfg(all(not(debug_assertions), feature = "sentry"))] - if !opt.no_sentry { - let logger = sentry::integrations::log::SentryLogger::with_dest(log_builder.build()); + #[cfg(all(not(debug_assertions), feature = "analytics"))] + if !opt.no_analytics { + let logger = + sentry::integrations::log::SentryLogger::with_dest(log_builder.build()); log::set_boxed_logger(Box::new(logger)) .map(|()| log::set_max_level(log::LevelFilter::Info)) .unwrap(); let sentry = sentry::init(sentry::ClientOptions { release: sentry::release_name!(), - dsn: Some(opt.sentry_dsn.parse()?), + dsn: Some(SENTRY_DSN.parse()?), ..Default::default() }); // sentry must stay alive as long as possible @@ -50,25 +55,14 @@ async fn main() -> Result<(), MainError> { _ => unreachable!(), } - //if let Some(path) = &opt.import_snapshot { - //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; - //} - let data = Data::new(opt.clone())?; - //if !opt.no_analytics { - //let analytics_data = data.clone(); - //let analytics_opt = opt.clone(); - //thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); - //} - - //if let Some(path) = &opt.import_dump { - //dump::import_dump(&data, path, opt.dump_batch_size)?; - //} - - //if opt.schedule_snapshot { - //snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; - //} + #[cfg(feature = "analytics")] + if !opt.no_analytics { + let analytics_data = data.clone(); + let analytics_opt = opt.clone(); + tokio::task::spawn(analytics::analytics_sender(analytics_data, analytics_opt)); + } print_launch_resume(&opt, &data); @@ -127,24 +121,21 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { env!("CARGO_PKG_VERSION").to_string() ); - #[cfg(all(not(debug_assertions), feature = "sentry"))] - eprintln!( - "Sentry DSN:\t\t{:?}", - if !opt.no_sentry { - &opt.sentry_dsn + #[cfg(feature = "analytics")] + { + if opt.no_analytics { + eprintln!("Anonymous telemetry:\t\"Disabled\""); } else { - "Disabled" - } - ); + eprintln!( + " +Thank you for using MeiliSearch! +We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics - eprintln!( - "Anonymous telemetry:\t{:?}", - if !opt.no_analytics { - "Enabled" - } else { - "Disabled" +Anonymous telemetry: \"Enabled\" +" + ); } - ); + } eprintln!(); diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index eb81ab9fd..54a50742a 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -96,21 +96,6 @@ pub struct Opt { #[structopt(long, env = "MEILI_MASTER_KEY")] pub master_key: Option, - /// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project. - /// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable. - #[cfg(all(not(debug_assertions), feature = "sentry"))] - #[structopt( - long, - env = "SENTRY_DSN", - default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337" - )] - pub sentry_dsn: String, - - /// Disable Sentry error reporting. - #[structopt(long, env = "MEILI_NO_SENTRY")] - #[cfg(all(not(debug_assertions), feature = "sentry"))] - pub no_sentry: bool, - /// This environment variable must be set to `production` if you are running in production. /// If the server is running in development mode more logs will be displayed, /// and the master key can be avoided which implies that there is no security on the updates routes. @@ -119,6 +104,7 @@ pub struct Opt { pub env: String, /// Do not send analytics to Meili. + #[cfg(feature = "analytics")] #[structopt(long, env = "MEILI_NO_ANALYTICS")] pub no_analytics: bool, diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 0fb801d7f..2c47a8ca8 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -71,6 +71,7 @@ pub fn default_settings(dir: impl AsRef) -> Opt { http_addr: "127.0.0.1:7700".to_owned(), master_key: None, env: "development".to_owned(), + #[cfg(feature = "analytics")] no_analytics: true, max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), @@ -90,9 +91,5 @@ pub fn default_settings(dir: impl AsRef) -> Opt { snapshot_interval_sec: 0, import_dump: None, indexer_options: IndexerOpts::default(), - #[cfg(all(not(debug_assertions), feature = "sentry"))] - sentry_dsn: String::from(""), - #[cfg(all(not(debug_assertions), feature = "sentry"))] - no_sentry: true, } } From 79046378937dbfb59d1c5280e6b8077aa0fe98ff Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 5 May 2021 17:31:40 +0200 Subject: [PATCH 395/522] crop skeleton --- meilisearch-http/src/index/search.rs | 62 ++++++++++++++++------------ 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 50d163898..21a587a37 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -232,7 +232,8 @@ fn compute_formatted>( highlighter: &Highlighter, matching_words: &impl Matcher, all_formatted: &[FieldId], - to_highlight_ids: &HashSet, + to_highlight_fields: &HashSet, + to_crop_fields: &HashSet, ) -> anyhow::Result { let mut document = Document::new(); @@ -240,8 +241,8 @@ fn compute_formatted>( if let Some(value) = obkv.get(*field) { let mut value: Value = serde_json::from_slice(value)?; - if to_highlight_ids.contains(field) { - value = highlighter.highlight_value(value, matching_words); + if to_highlight_fields.contains(field) { + value = highlighter.format_value(value, matching_words, to_highlight_fields.contains(field)); } // This unwrap must be safe since we got the ids from the fields_ids_map just @@ -291,46 +292,55 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { Self { analyzer, marks } } - fn highlight_value(&self, value: Value, words_to_highlight: &impl Matcher) -> Value { + fn format_value( + &self, + value: Value, + matcher: &impl Matcher, + need_to_crop: Option, + need_to_highlight: bool, + ) -> Value { match value { - Value::Null => Value::Null, - Value::Bool(boolean) => Value::Bool(boolean), - Value::Number(number) => Value::Number(number), Value::String(old_string) => { - let mut string = String::new(); - let analyzed = self.analyzer.analyze(&old_string); - for (word, token) in analyzed.reconstruct() { - if token.is_word() { - let to_highlight = words_to_highlight.matches(token.text()); - if to_highlight { - string.push_str(&self.marks.0) - } - string.push_str(word); - if to_highlight { - string.push_str(&self.marks.1) - } - } else { - string.push_str(word); - } - } - Value::String(string) + let value = self.format_string(old_string, need_to_crop, need_to_highlight); + Value::String(value) } Value::Array(values) => Value::Array( values .into_iter() - .map(|v| self.highlight_value(v, words_to_highlight)) + .map(|v| self.format_value(v, matcher, None, need_to_highlight)) .collect(), ), Value::Object(object) => Value::Object( object .into_iter() - .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) + .map(|(k, v)| (k, self.format_value(value, matcher, None, need_to_highlight))) .collect(), ), + value => value, } } + fn format_string(&self, s: String, need_to_crop: Option, need_to_highlight: bool) -> String { + let word_iter: Box> = if let Some(_crop_len) = need_to_crop { + // cropping iterator + todo!() + } else { + // normal Iterator + todo!() + }; + + word_iter.map(|(word, is_match)| { + if need_to_highlight && is_match { + // highlight word + todo!() + } else { + word + } + }) + .collect::() + } } + fn parse_facets( facets: &Value, index: &Index, From a03d9d496e2cd85c22e74811b4103053e850cb65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 6 May 2021 16:32:11 +0200 Subject: [PATCH 396/522] Fix compilation errors --- meilisearch-http/src/index/search.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 21a587a37..262535888 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -168,6 +168,7 @@ impl Index { &matching_words, all_formatted.as_ref().as_slice(), &to_highlight_ids, + &to_crop_ids, )?; let hit = SearchHit { document, @@ -241,8 +242,14 @@ fn compute_formatted>( if let Some(value) = obkv.get(*field) { let mut value: Value = serde_json::from_slice(value)?; + let need_to_crop = if to_crop_fields.contains(field) { + Some(200) // TO CHANGE + } else { + None + }; + if to_highlight_fields.contains(field) { - value = highlighter.format_value(value, matching_words, to_highlight_fields.contains(field)); + value = highlighter.format_value(value, matching_words, need_to_crop, to_highlight_fields.contains(field)); } // This unwrap must be safe since we got the ids from the fields_ids_map just @@ -313,7 +320,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { Value::Object(object) => Value::Object( object .into_iter() - .map(|(k, v)| (k, self.format_value(value, matcher, None, need_to_highlight))) + .map(|(k, v)| (k, self.format_value(v, matcher, None, need_to_highlight))) .collect(), ), value => value, From 60f6d1c373c1c4319bc982b3061f98f6cea67097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 6 May 2021 18:31:41 +0200 Subject: [PATCH 397/522] First version of highlight after refacto --- meilisearch-http/src/index/search.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 262535888..f9b1508f9 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -308,7 +308,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { ) -> Value { match value { Value::String(old_string) => { - let value = self.format_string(old_string, need_to_crop, need_to_highlight); + let value = self.format_string(old_string, matcher, need_to_crop, need_to_highlight); Value::String(value) } Value::Array(values) => Value::Array( @@ -326,19 +326,28 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { value => value, } } - fn format_string(&self, s: String, need_to_crop: Option, need_to_highlight: bool) -> String { + fn format_string(&self, s: String, matcher: &impl Matcher, need_to_crop: Option, need_to_highlight: bool) -> String { + let analyzed = self.analyzer.analyze(&s); let word_iter: Box> = if let Some(_crop_len) = need_to_crop { // cropping iterator todo!() } else { - // normal Iterator - todo!() + Box::new(analyzed.reconstruct().map(|(word, token)| { + if token.is_word() && matcher.matches(token.text()){ + (word.to_string(), true) + } else { + (word.to_string(), false) + } + })) }; word_iter.map(|(word, is_match)| { if need_to_highlight && is_match { - // highlight word - todo!() + let mut new_word = String::new(); + new_word.push_str(&self.marks.0); + new_word.push_str(&word); + new_word.push_str(&self.marks.1); + new_word } else { word } From 93002e734cebabfb63c9f66b16e4dac1a3ea6a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 6 May 2021 18:41:04 +0200 Subject: [PATCH 398/522] Fix tests --- meilisearch-http/src/index/search.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index f9b1508f9..df553d6ef 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -423,6 +423,7 @@ mod test { let all_formatted = Vec::new(); let to_highlight_ids = HashSet::new(); + let to_crop_ids = HashSet::new(); let matching_words = MatchingWords::default(); @@ -433,8 +434,8 @@ mod test { &matching_words, &all_formatted, &to_highlight_ids, - ) - .unwrap(); + &to_crop_ids, + ).unwrap(); assert!(value.is_empty()); } @@ -458,6 +459,7 @@ mod test { let all_formatted = vec![id]; let to_highlight_ids = HashSet::new(); + let to_crop_ids = HashSet::new(); let matching_words = MatchingWords::default(); @@ -468,8 +470,8 @@ mod test { &matching_words, &all_formatted, &to_highlight_ids, - ) - .unwrap(); + &to_crop_ids, + ).unwrap(); assert_eq!(value["test"], "hello"); } @@ -493,6 +495,7 @@ mod test { let all_formatted = vec![id]; let to_highlight_ids = HashSet::from_iter(Some(id)); + let to_crop_ids = HashSet::new(); let matching_words = HashSet::from_iter(Some(String::from("hello"))); @@ -503,8 +506,8 @@ mod test { &matching_words, &all_formatted, &to_highlight_ids, - ) - .unwrap(); + &to_crop_ids, + ).unwrap(); assert_eq!(value["test"], "hello"); } From 56c9633c53c4369b5804569ef824cd977fb1cc12 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 11 May 2021 17:27:31 +0200 Subject: [PATCH 399/522] simple crop before --- meilisearch-http/src/index/search.rs | 38 +++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index df553d6ef..ce9338d5f 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -7,7 +7,7 @@ use either::Either; use heed::RoTxn; use indexmap::IndexMap; use itertools::Itertools; -use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token}; use milli::{FilterCondition, FieldId, FieldsIdsMap, MatchingWords}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -303,7 +303,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { &self, value: Value, matcher: &impl Matcher, - need_to_crop: Option, + need_to_crop: Option, need_to_highlight: bool, ) -> Value { match value { @@ -326,30 +326,34 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { value => value, } } - fn format_string(&self, s: String, matcher: &impl Matcher, need_to_crop: Option, need_to_highlight: bool) -> String { + fn format_string(&self, s: String, matcher: &impl Matcher, need_to_crop: Option, need_to_highlight: bool) -> String { let analyzed = self.analyzer.analyze(&s); - let word_iter: Box> = if let Some(_crop_len) = need_to_crop { - // cropping iterator - todo!() - } else { - Box::new(analyzed.reconstruct().map(|(word, token)| { - if token.is_word() && matcher.matches(token.text()){ - (word.to_string(), true) - } else { - (word.to_string(), false) - } - })) + + let tokens: Box> = match need_to_crop { + Some(crop_len) => { + let mut taken = 0; + let iter = analyzed + .reconstruct() + .skip_while(|(_, token)| !matcher.matches(token.text())) + .take_while(move |(word, _)| { + let take = taken < crop_len; + taken += word.chars().count(); + take + }); + Box::new(iter) + }, + None => Box::new(analyzed.reconstruct()), }; - word_iter.map(|(word, is_match)| { - if need_to_highlight && is_match { + tokens.map(|(word, token)| { + if need_to_highlight && token.is_word() && matcher.matches(token.text()){ let mut new_word = String::new(); new_word.push_str(&self.marks.0); new_word.push_str(&word); new_word.push_str(&self.marks.1); new_word } else { - word + word.to_string() } }) .collect::() From 7473cc6e27fe2658ee48f90afe4656042a6826ac Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Tue, 11 May 2021 18:30:55 +0200 Subject: [PATCH 400/522] implement crop around --- meilisearch-http/src/index/search.rs | 126 +++++++++++++++++---------- 1 file changed, 79 insertions(+), 47 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index ce9338d5f..760357b9a 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,6 +1,6 @@ -use std::borrow::Cow; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashSet, VecDeque}; use std::time::Instant; +use std::{borrow::Cow, collections::HashMap}; use anyhow::bail; use either::Either; @@ -157,7 +157,12 @@ impl Index { let stop_words = fst::Set::default(); let highlighter = - Highlighter::new(&stop_words, (String::from(""), String::from(""))); + Formatter::new(&stop_words, (String::from(""), String::from(""))); + + let to_crop = to_crop_ids + .into_iter() + .map(|id| (id, query.crop_length)) + .collect::>(); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let document = make_document(&all_attributes, &fields_ids_map, obkv)?; @@ -168,7 +173,7 @@ impl Index { &matching_words, all_formatted.as_ref().as_slice(), &to_highlight_ids, - &to_crop_ids, + &to_crop, )?; let hit = SearchHit { document, @@ -230,11 +235,11 @@ fn make_document( fn compute_formatted>( field_ids_map: &FieldsIdsMap, obkv: obkv::KvReader, - highlighter: &Highlighter, + highlighter: &Formatter, matching_words: &impl Matcher, all_formatted: &[FieldId], to_highlight_fields: &HashSet, - to_crop_fields: &HashSet, + to_crop_fields: &HashMap>, ) -> anyhow::Result { let mut document = Document::new(); @@ -242,15 +247,12 @@ fn compute_formatted>( if let Some(value) = obkv.get(*field) { let mut value: Value = serde_json::from_slice(value)?; - let need_to_crop = if to_crop_fields.contains(field) { - Some(200) // TO CHANGE - } else { - None - }; - - if to_highlight_fields.contains(field) { - value = highlighter.format_value(value, matching_words, need_to_crop, to_highlight_fields.contains(field)); - } + value = highlighter.format_value( + value, + matching_words, + to_crop_fields.get(field).copied().flatten(), + to_highlight_fields.contains(field), + ); // This unwrap must be safe since we got the ids from the fields_ids_map just // before. @@ -284,12 +286,12 @@ impl Matcher for MatchingWords { } } -struct Highlighter<'a, A> { +struct Formatter<'a, A> { analyzer: Analyzer<'a, A>, marks: (String, String), } -impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { +impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { pub fn new(stop_words: &'a fst::Set, marks: (String, String)) -> Self { let mut config = AnalyzerConfig::default(); config.stop_words(stop_words); @@ -305,10 +307,11 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { matcher: &impl Matcher, need_to_crop: Option, need_to_highlight: bool, - ) -> Value { + ) -> Value { match value { Value::String(old_string) => { - let value = self.format_string(old_string, matcher, need_to_crop, need_to_highlight); + let value = + self.format_string(old_string, matcher, need_to_crop, need_to_highlight); Value::String(value) } Value::Array(values) => Value::Array( @@ -326,41 +329,67 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { value => value, } } - fn format_string(&self, s: String, matcher: &impl Matcher, need_to_crop: Option, need_to_highlight: bool) -> String { + fn format_string( + &self, + s: String, + matcher: &impl Matcher, + need_to_crop: Option, + need_to_highlight: bool, + ) -> String { let analyzed = self.analyzer.analyze(&s); - let tokens: Box> = match need_to_crop { + let tokens: Box> = match need_to_crop { Some(crop_len) => { - let mut taken = 0; - let iter = analyzed - .reconstruct() - .skip_while(|(_, token)| !matcher.matches(token.text())) + let mut buffer = VecDeque::new(); + let mut tokens = analyzed.reconstruct().peekable(); + let mut taken_before = 0; + while let Some((word, token)) = tokens.next_if(|(_, token)| !matcher.matches(token.text())) { + buffer.push_back((word, token)); + taken_before += word.chars().count(); + while taken_before > crop_len { + if let Some((word, _)) = buffer.pop_front() { + taken_before -= word.chars().count(); + } + } + } + + if let Some(token) = tokens.next() { + buffer.push_back(token); + } + + let mut taken_after = 0; + + let after_iter = tokens .take_while(move |(word, _)| { - let take = taken < crop_len; - taken += word.chars().count(); + let take = taken_after <= crop_len; + taken_after += word.chars().count(); take }); + let iter = buffer + .into_iter() + .chain(after_iter); + Box::new(iter) - }, + } None => Box::new(analyzed.reconstruct()), }; - tokens.map(|(word, token)| { - if need_to_highlight && token.is_word() && matcher.matches(token.text()){ - let mut new_word = String::new(); - new_word.push_str(&self.marks.0); - new_word.push_str(&word); - new_word.push_str(&self.marks.1); - new_word - } else { - word.to_string() - } - }) - .collect::() + tokens + .map(|(word, token)| { + if need_to_highlight && token.is_word() && matcher.matches(token.text()) { + let mut new_word = String::new(); + new_word.push_str(&self.marks.0); + new_word.push_str(&word); + new_word.push_str(&self.marks.1); + new_word + } else { + word.to_string() + } + }) + .collect::() } } - fn parse_facets( facets: &Value, index: &Index, @@ -412,7 +441,7 @@ mod test { fn no_formatted() { let stop_words = fst::Set::default(); let highlighter = - Highlighter::new(&stop_words, (String::from(""), String::from(""))); + Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let id = fields.insert("test").unwrap(); @@ -439,7 +468,8 @@ mod test { &all_formatted, &to_highlight_ids, &to_crop_ids, - ).unwrap(); + ) + .unwrap(); assert!(value.is_empty()); } @@ -448,7 +478,7 @@ mod test { fn formatted_no_highlight() { let stop_words = fst::Set::default(); let highlighter = - Highlighter::new(&stop_words, (String::from(""), String::from(""))); + Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let id = fields.insert("test").unwrap(); @@ -475,7 +505,8 @@ mod test { &all_formatted, &to_highlight_ids, &to_crop_ids, - ).unwrap(); + ) + .unwrap(); assert_eq!(value["test"], "hello"); } @@ -484,7 +515,7 @@ mod test { fn formatted_with_highlight() { let stop_words = fst::Set::default(); let highlighter = - Highlighter::new(&stop_words, (String::from(""), String::from(""))); + Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let id = fields.insert("test").unwrap(); @@ -511,7 +542,8 @@ mod test { &all_formatted, &to_highlight_ids, &to_crop_ids, - ).unwrap(); + ) + .unwrap(); assert_eq!(value["test"], "hello"); } From caaf8d3f4077317d497bd777379a3441b00ddc5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 3 Jun 2021 17:54:53 +0200 Subject: [PATCH 401/522] Fix tests --- meilisearch-http/src/index/search.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 760357b9a..eb6f98d87 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -456,7 +456,7 @@ mod test { let all_formatted = Vec::new(); let to_highlight_ids = HashSet::new(); - let to_crop_ids = HashSet::new(); + let to_crop_ids = HashMap::new(); let matching_words = MatchingWords::default(); @@ -493,7 +493,7 @@ mod test { let all_formatted = vec![id]; let to_highlight_ids = HashSet::new(); - let to_crop_ids = HashSet::new(); + let to_crop_ids = HashMap::new(); let matching_words = MatchingWords::default(); @@ -530,7 +530,7 @@ mod test { let all_formatted = vec![id]; let to_highlight_ids = HashSet::from_iter(Some(id)); - let to_crop_ids = HashSet::new(); + let to_crop_ids = HashMap::new(); let matching_words = HashSet::from_iter(Some(String::from("hello"))); From 811bc2f421567cfe7b8cfc2a2fb5e92f770ae580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Fri, 4 Jun 2021 02:25:38 +0200 Subject: [PATCH 402/522] Around to previous word --- meilisearch-http/src/index/search.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index eb6f98d87..429b5582a 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -16,8 +16,12 @@ use super::Index; pub type Document = IndexMap; -pub const DEFAULT_SEARCH_LIMIT: usize = 20; +// pub const DEFAULT_CROP_LENGTH: usize = 5; +// const fn default_crop_length() -> Option { +// Some(DEFAULT_CROP_LENGTH) +// } +pub const DEFAULT_SEARCH_LIMIT: usize = 20; const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } @@ -31,6 +35,7 @@ pub struct SearchQuery { pub limit: usize, pub attributes_to_retrieve: Option>, pub attributes_to_crop: Option>, + // #[serde(default = "default_crop_length")] pub crop_length: Option, pub attributes_to_highlight: Option>, pub matches: Option, @@ -162,6 +167,7 @@ impl Index { let to_crop = to_crop_ids .into_iter() .map(|id| (id, query.crop_length)) + // .map(|id| (id, Some(5))) .collect::>(); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { @@ -346,7 +352,13 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { while let Some((word, token)) = tokens.next_if(|(_, token)| !matcher.matches(token.text())) { buffer.push_back((word, token)); taken_before += word.chars().count(); - while taken_before > crop_len { + while taken_before >= crop_len { + // Around to the previous word + if let Some((word, _)) = buffer.front() { + if taken_before - word.chars().count() < crop_len { + break; + } + } if let Some((word, _)) = buffer.pop_front() { taken_before -= word.chars().count(); } @@ -358,13 +370,13 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { } let mut taken_after = 0; - let after_iter = tokens .take_while(move |(word, _)| { - let take = taken_after <= crop_len; + let take = taken_after < crop_len; taken_after += word.chars().count(); take }); + let iter = buffer .into_iter() .chain(after_iter); From 0da8fa115e90302e3ccbd62dccd6a94b2bfc8272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 8 Jun 2021 17:33:20 +0200 Subject: [PATCH 403/522] Add custom croplength for attributes to crop --- meilisearch-http/src/index/search.rs | 61 +++++++++++++++++++-------- meilisearch-http/src/routes/search.rs | 2 +- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 429b5582a..603ebf1b3 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -16,16 +16,16 @@ use super::Index; pub type Document = IndexMap; -// pub const DEFAULT_CROP_LENGTH: usize = 5; -// const fn default_crop_length() -> Option { -// Some(DEFAULT_CROP_LENGTH) -// } - pub const DEFAULT_SEARCH_LIMIT: usize = 20; const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } +pub const DEFAULT_CROP_LENGTH: usize = 200; +const fn default_crop_length() -> Option { + Some(DEFAULT_CROP_LENGTH) +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchQuery { @@ -34,8 +34,8 @@ pub struct SearchQuery { #[serde(default = "default_search_limit")] pub limit: usize, pub attributes_to_retrieve: Option>, - pub attributes_to_crop: Option>, - // #[serde(default = "default_crop_length")] + pub attributes_to_crop: Option>, + #[serde(default = "default_crop_length")] pub crop_length: Option, pub attributes_to_highlight: Option>, pub matches: Option, @@ -126,12 +126,45 @@ impl Index { .map(fids) .unwrap_or_default(); - let to_crop_ids = query + let to_crop_ids_length = query .attributes_to_crop .as_ref() - .map(fids) + .map(|attributes: &Vec| { + let mut ids_length_crop = HashMap::new(); + for attribute in attributes { + let mut attr_name = attribute.clone(); + let mut attr_len = query.crop_length; + + if attr_name.contains(":") { + let mut split = attr_name.rsplit(':'); + attr_len = match split.nth(0) { + Some(s) => s.parse::().ok(), + None => None, + }; + attr_name = split.flat_map(|s| s.chars()).collect(); + } + + if attr_name == "*" { + let ids = displayed_ids.clone(); + for id in ids { + ids_length_crop.insert(id, attr_len); + } + } + + if let Some(id) = fields_ids_map.id(&attr_name) { + ids_length_crop.insert(id, attr_len); + } + } + ids_length_crop + }) .unwrap_or_default(); + let to_crop_ids = to_crop_ids_length + .clone() + .into_iter() + .map(|(k, _)| k) + .collect::>(); + // The attributes to retrieve are: // - the ones explicitly marked as to retrieve that are also in the displayed attributes let all_attributes: Vec<_> = to_retrieve_ids @@ -164,12 +197,6 @@ impl Index { let highlighter = Formatter::new(&stop_words, (String::from(""), String::from(""))); - let to_crop = to_crop_ids - .into_iter() - .map(|id| (id, query.crop_length)) - // .map(|id| (id, Some(5))) - .collect::>(); - for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let document = make_document(&all_attributes, &fields_ids_map, obkv)?; let formatted = compute_formatted( @@ -179,7 +206,7 @@ impl Index { &matching_words, all_formatted.as_ref().as_slice(), &to_highlight_ids, - &to_crop, + &to_crop_ids_length, )?; let hit = SearchHit { document, @@ -352,7 +379,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { while let Some((word, token)) = tokens.next_if(|(_, token)| !matcher.matches(token.text())) { buffer.push_back((word, token)); taken_before += word.chars().count(); - while taken_before >= crop_len { + while taken_before > crop_len { // Around to the previous word if let Some((word, _)) = buffer.front() { if taken_before - word.chars().count() < crop_len { diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index be06960cf..8489215e7 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -40,7 +40,7 @@ impl TryFrom for SearchQuery { let attributes_to_crop = other .attributes_to_crop - .map(|attrs| attrs.split(',').map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let attributes_to_highlight = other .attributes_to_highlight From 9e69f33f3c3b9792cc85314bc786440fd1a16bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 8 Jun 2021 18:02:04 +0200 Subject: [PATCH 404/522] Fix clippy errors --- meilisearch-http/src/index/search.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 603ebf1b3..bedde0997 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -135,9 +135,9 @@ impl Index { let mut attr_name = attribute.clone(); let mut attr_len = query.crop_length; - if attr_name.contains(":") { + if attr_name.contains(':') { let mut split = attr_name.rsplit(':'); - attr_len = match split.nth(0) { + attr_len = match split.next() { Some(s) => s.parse::().ok(), None => None, }; From 4f8c771bb567ed96b14758e8d836911efa4b528a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Sun, 13 Jun 2021 11:53:29 +0200 Subject: [PATCH 405/522] Add new line --- meilisearch-http/src/index/search.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index bedde0997..ae476318e 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -362,6 +362,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { value => value, } } + fn format_string( &self, s: String, From 7f84f59472dcab518e215e97904d3144a9d9b9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Sun, 13 Jun 2021 12:00:38 +0200 Subject: [PATCH 406/522] Reorganize imports --- meilisearch-http/src/index/search.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index ae476318e..47a9bd4fb 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,6 +1,6 @@ -use std::collections::{BTreeMap, HashSet, VecDeque}; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::time::Instant; -use std::{borrow::Cow, collections::HashMap}; use anyhow::bail; use either::Either; From 638009fb2b31d9970b28ce642f1a78a8b2f658b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Sun, 13 Jun 2021 12:29:24 +0200 Subject: [PATCH 407/522] Rename highlighter variable into formatter --- meilisearch-http/src/index/search.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 47a9bd4fb..acabcf46d 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -194,7 +194,7 @@ impl Index { }; let stop_words = fst::Set::default(); - let highlighter = + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { @@ -202,7 +202,7 @@ impl Index { let formatted = compute_formatted( &fields_ids_map, obkv, - &highlighter, + &formatter, &matching_words, all_formatted.as_ref().as_slice(), &to_highlight_ids, @@ -268,7 +268,7 @@ fn make_document( fn compute_formatted>( field_ids_map: &FieldsIdsMap, obkv: obkv::KvReader, - highlighter: &Formatter, + formatter: &Formatter, matching_words: &impl Matcher, all_formatted: &[FieldId], to_highlight_fields: &HashSet, @@ -280,7 +280,7 @@ fn compute_formatted>( if let Some(value) = obkv.get(*field) { let mut value: Value = serde_json::from_slice(value)?; - value = highlighter.format_value( + value = formatter.format_value( value, matching_words, to_crop_fields.get(field).copied().flatten(), @@ -301,7 +301,7 @@ fn compute_formatted>( Ok(document) } -/// trait to allow unit testing of `compute_formated` +/// trait to allow unit testing of `compute_formatted` trait Matcher { fn matches(&self, w: &str) -> bool; } @@ -480,7 +480,7 @@ mod test { #[test] fn no_formatted() { let stop_words = fst::Set::default(); - let highlighter = + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); @@ -503,7 +503,7 @@ mod test { let value = compute_formatted( &fields, obkv, - &highlighter, + &formatter, &matching_words, &all_formatted, &to_highlight_ids, @@ -517,7 +517,7 @@ mod test { #[test] fn formatted_no_highlight() { let stop_words = fst::Set::default(); - let highlighter = + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); @@ -540,7 +540,7 @@ mod test { let value = compute_formatted( &fields, obkv, - &highlighter, + &formatter, &matching_words, &all_formatted, &to_highlight_ids, @@ -554,7 +554,7 @@ mod test { #[test] fn formatted_with_highlight() { let stop_words = fst::Set::default(); - let highlighter = + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); @@ -577,7 +577,7 @@ mod test { let value = compute_formatted( &fields, obkv, - &highlighter, + &formatter, &matching_words, &all_formatted, &to_highlight_ids, From 65130d9ee7aa380dc42a404032bf50aa79eb80ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Sun, 13 Jun 2021 12:37:38 +0200 Subject: [PATCH 408/522] Change crop_length type from Option(usize) to usize --- meilisearch-http/src/index/search.rs | 8 ++++---- meilisearch-http/src/routes/search.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index acabcf46d..7e93b7099 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -22,8 +22,8 @@ const fn default_search_limit() -> usize { } pub const DEFAULT_CROP_LENGTH: usize = 200; -const fn default_crop_length() -> Option { - Some(DEFAULT_CROP_LENGTH) +const fn default_crop_length() -> usize { + DEFAULT_CROP_LENGTH } #[derive(Deserialize)] @@ -36,7 +36,7 @@ pub struct SearchQuery { pub attributes_to_retrieve: Option>, pub attributes_to_crop: Option>, #[serde(default = "default_crop_length")] - pub crop_length: Option, + pub crop_length: usize, pub attributes_to_highlight: Option>, pub matches: Option, pub filter: Option, @@ -133,7 +133,7 @@ impl Index { let mut ids_length_crop = HashMap::new(); for attribute in attributes { let mut attr_name = attribute.clone(); - let mut attr_len = query.crop_length; + let mut attr_len = Some(query.crop_length); if attr_name.contains(':') { let mut split = attr_name.rsplit(':'); diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 8489215e7..c2c83e3c8 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -23,7 +23,7 @@ pub struct SearchQueryGet { limit: Option, attributes_to_retrieve: Option, attributes_to_crop: Option, - crop_length: Option, + crop_length: usize, attributes_to_highlight: Option, filter: Option, matches: Option, From d0ec081e4900171ecf103c21a45b49bd7bdf2505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Sun, 13 Jun 2021 23:51:33 +0200 Subject: [PATCH 409/522] Refacto --- meilisearch-http/src/index/search.rs | 492 +++++++++++++++++++++------ 1 file changed, 380 insertions(+), 112 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 7e93b7099..b2c606c02 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,4 +1,4 @@ -use std::borrow::Cow; +// use std::borrow::Cow; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::time::Instant; @@ -65,6 +65,12 @@ pub struct SearchResult { pub facet_distributions: Option>>, } +#[derive(Copy, Clone)] +struct FormatOptions { + highlight: bool, + crop: Option, +} + impl Index { pub fn perform_search(&self, query: SearchQuery) -> anyhow::Result { let before_search = Instant::now(); @@ -108,7 +114,9 @@ impl Index { } if let Some(id) = fields_ids_map.id(attr) { - ids.insert(id); + if displayed_ids.contains(&id) { + ids.insert(id); + } } } ids @@ -120,51 +128,6 @@ impl Index { .map(fids) .unwrap_or_else(|| displayed_ids.clone()); - let to_highlight_ids = query - .attributes_to_highlight - .as_ref() - .map(fids) - .unwrap_or_default(); - - let to_crop_ids_length = query - .attributes_to_crop - .as_ref() - .map(|attributes: &Vec| { - let mut ids_length_crop = HashMap::new(); - for attribute in attributes { - let mut attr_name = attribute.clone(); - let mut attr_len = Some(query.crop_length); - - if attr_name.contains(':') { - let mut split = attr_name.rsplit(':'); - attr_len = match split.next() { - Some(s) => s.parse::().ok(), - None => None, - }; - attr_name = split.flat_map(|s| s.chars()).collect(); - } - - if attr_name == "*" { - let ids = displayed_ids.clone(); - for id in ids { - ids_length_crop.insert(id, attr_len); - } - } - - if let Some(id) = fields_ids_map.id(&attr_name) { - ids_length_crop.insert(id, attr_len); - } - } - ids_length_crop - }) - .unwrap_or_default(); - - let to_crop_ids = to_crop_ids_length - .clone() - .into_iter() - .map(|(k, _)| k) - .collect::>(); - // The attributes to retrieve are: // - the ones explicitly marked as to retrieve that are also in the displayed attributes let all_attributes: Vec<_> = to_retrieve_ids @@ -173,25 +136,152 @@ impl Index { .sorted() .collect(); + let mut formatted_options = HashMap::new(); + + let attr_to_highlight = query.attributes_to_highlight.unwrap_or_default(); + for attr in attr_to_highlight { + let new_format = FormatOptions { + highlight: true, + crop: None, + }; + + if attr == "*" { + let ids = displayed_ids.clone(); + for id in ids { + formatted_options.insert(id, new_format); + } + break; + } + + if let Some(id) = fields_ids_map.id(&attr) { + if displayed_ids.contains(&id) { + formatted_options.insert(id, new_format); + } + } + }; + + let attr_to_crop = query.attributes_to_crop.unwrap_or_default(); + for attr in attr_to_crop { + let mut attr_name = attr.clone(); + let mut attr_len = Some(query.crop_length); + + if attr_name.contains(':') { + let mut split = attr_name.rsplit(':'); + attr_len = match split.next() { + Some(s) => s.parse::().ok(), + None => None, + }; + attr_name = split.flat_map(|s| s.chars()).collect(); + } + + if attr_name == "*" { + let ids = displayed_ids.clone(); + for id in ids { + let mut highlight = false; + if let Some(f) = formatted_options.get(&id) { + highlight = f.highlight; + } + formatted_options.insert(id, FormatOptions { + highlight: highlight, + crop: attr_len, + }); + } + } + + if let Some(id) = fields_ids_map.id(&attr_name) { + if displayed_ids.contains(&id) { + let mut highlight = false; + if let Some(f) = formatted_options.get(&id) { + highlight = f.highlight; + } + formatted_options.insert(id, FormatOptions { + highlight: highlight, + crop: attr_len, + }); + } + } + } + + let formatted_ids = formatted_options + .keys() + .cloned() + .collect::>(); + + // All attributes present in `_formatted` that are not necessary highighted or croped + let ids_in_formatted = formatted_ids + .union(&to_retrieve_ids) + .cloned() + .sorted() + .collect::>(); + + + // let to_highlight_ids = query // PLUS BESOIN + // .attributes_to_highlight + // .as_ref() + // .map(fids) + // .unwrap_or_default(); + + + // let to_crop_ids_length = query + // .attributes_to_crop + // .as_ref() + // .map(|attributes: &Vec| { + // let mut ids_length_crop = HashMap::new(); + // for attribute in attributes { + // let mut attr_name = attribute.clone(); + // let mut attr_len = Some(query.crop_length); + + // if attr_name.contains(':') { + // let mut split = attr_name.rsplit(':'); + // attr_len = match split.next() { + // Some(s) => s.parse::().ok(), + // None => None, + // }; + // attr_name = split.flat_map(|s| s.chars()).collect(); + // } + + // if attr_name == "*" { + // let ids = displayed_ids.clone(); + // for id in ids { + // ids_length_crop.insert(id, attr_len); + // } + // } + + // if let Some(id) = fields_ids_map.id(&attr_name) { + // if displayed_ids.contains(&id) { + // ids_length_crop.insert(id, attr_len); + // } + // } + // } + // ids_length_crop + // }) + // .unwrap_or_default(); + + // let to_crop_ids = to_crop_ids_length // PLUS BESOIN + // .clone() + // .into_iter() + // .map(|(k, _)| k) + // .collect::>(); + // The formatted attributes are: // - The one in either highlighted attributes or cropped attributes if there are attributes // to retrieve // - All the attributes to retrieve if there are either highlighted or cropped attributes // the request specified that all attributes are to retrieve (i.e attributes to retrieve is // empty in the query) - let all_formatted = if query.attributes_to_retrieve.is_none() { - if query.attributes_to_highlight.is_some() || query.attributes_to_crop.is_some() { - Cow::Borrowed(&all_attributes) - } else { - Cow::Owned(Vec::new()) - } - } else { - let attrs = (&to_crop_ids | &to_highlight_ids) - .intersection(&displayed_ids) - .cloned() - .collect::>(); - Cow::Owned(attrs) - }; + // let all_formatted = if query.attributes_to_retrieve.is_none() { + // if query.attributes_to_highlight.is_some() || query.attributes_to_crop.is_some() { + // Cow::Borrowed(&all_attributes) + // } else { + // Cow::Owned(Vec::new()) + // } + // } else { + // let attrs = (&to_crop_ids | &to_highlight_ids) + // .intersection(&displayed_ids) + // .cloned() + // .collect::>(); + // Cow::Owned(attrs) + // }; let stop_words = fst::Set::default(); let formatter = @@ -204,9 +294,11 @@ impl Index { obkv, &formatter, &matching_words, - all_formatted.as_ref().as_slice(), - &to_highlight_ids, - &to_crop_ids_length, + &ids_in_formatted, + // all_formatted.as_ref().as_slice(), + &formatted_options, + // &to_highlight_ids, //ICI + // &to_crop_ids_length, //ICI )?; let hit = SearchHit { document, @@ -270,31 +362,38 @@ fn compute_formatted>( obkv: obkv::KvReader, formatter: &Formatter, matching_words: &impl Matcher, - all_formatted: &[FieldId], - to_highlight_fields: &HashSet, - to_crop_fields: &HashMap>, + ids_in_formatted: &Vec, + formatted_options: &HashMap, + // to_highlight_fields: &HashSet, //ICI + // to_crop_fields: &HashMap>, //ICI ) -> anyhow::Result { let mut document = Document::new(); - for field in all_formatted { - if let Some(value) = obkv.get(*field) { - let mut value: Value = serde_json::from_slice(value)?; + if formatted_options.len() > 0 { + for field in ids_in_formatted { + if let Some(value) = obkv.get(*field) { + let mut value: Value = serde_json::from_slice(value)?; - value = formatter.format_value( - value, - matching_words, - to_crop_fields.get(field).copied().flatten(), - to_highlight_fields.contains(field), - ); + if let Some(format) = formatted_options.get(field) { + value = formatter.format_value( + value, + matching_words, + format.highlight, + format.crop, + // to_crop_fields.get(field).copied().flatten(), //ICI + // to_highlight_fields.contains(field), //ICI + ); + } - // This unwrap must be safe since we got the ids from the fields_ids_map just - // before. - let key = field_ids_map - .name(*field) - .expect("Missing field name") - .to_string(); + // This unwrap must be safe since we got the ids from the fields_ids_map just + // before. + let key = field_ids_map + .name(*field) + .expect("Missing field name") + .to_string(); - document.insert(key, value); + document.insert(key, value); + } } } @@ -338,25 +437,25 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { &self, value: Value, matcher: &impl Matcher, - need_to_crop: Option, need_to_highlight: bool, + need_to_crop: Option, ) -> Value { match value { Value::String(old_string) => { let value = - self.format_string(old_string, matcher, need_to_crop, need_to_highlight); + self.format_string(old_string, matcher, need_to_highlight, need_to_crop); Value::String(value) } Value::Array(values) => Value::Array( values .into_iter() - .map(|v| self.format_value(v, matcher, None, need_to_highlight)) + .map(|v| self.format_value(v, matcher, need_to_highlight, None)) .collect(), ), Value::Object(object) => Value::Object( object .into_iter() - .map(|(k, v)| (k, self.format_value(v, matcher, None, need_to_highlight))) + .map(|(k, v)| (k, self.format_value(v, matcher, need_to_highlight, None))) .collect(), ), value => value, @@ -367,8 +466,8 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { &self, s: String, matcher: &impl Matcher, - need_to_crop: Option, need_to_highlight: bool, + need_to_crop: Option, ) -> String { let analyzed = self.analyzer.analyze(&s); @@ -478,7 +577,7 @@ mod test { use super::*; #[test] - fn no_formatted() { + fn no_ids_no_formatted() { let stop_words = fst::Set::default(); let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); @@ -494,9 +593,8 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let all_formatted = Vec::new(); - let to_highlight_ids = HashSet::new(); - let to_crop_ids = HashMap::new(); + let ids_in_formatted = Vec::new(); + let formatted_options = HashMap::new(); let matching_words = MatchingWords::default(); @@ -505,9 +603,8 @@ mod test { obkv, &formatter, &matching_words, - &all_formatted, - &to_highlight_ids, - &to_crop_ids, + &ids_in_formatted, + &formatted_options, ) .unwrap(); @@ -515,7 +612,7 @@ mod test { } #[test] - fn formatted_no_highlight() { + fn no_formatted_with_ids() { let stop_words = fst::Set::default(); let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); @@ -531,9 +628,8 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let all_formatted = vec![id]; - let to_highlight_ids = HashSet::new(); - let to_crop_ids = HashMap::new(); + let ids_in_formatted = vec![id]; + let formatted_options = HashMap::new(); let matching_words = MatchingWords::default(); @@ -542,13 +638,12 @@ mod test { obkv, &formatter, &matching_words, - &all_formatted, - &to_highlight_ids, - &to_crop_ids, + &ids_in_formatted, + &formatted_options, ) .unwrap(); - assert_eq!(value["test"], "hello"); + assert!(value.is_empty()); } #[test] @@ -558,33 +653,206 @@ mod test { Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); - let id = fields.insert("test").unwrap(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()) + obkv.insert(title, Value::String("The Hobbit".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(author, Value::String("J. R. R. Tolkien".into()).to_string().as_bytes()) .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); - let all_formatted = vec![id]; - let to_highlight_ids = HashSet::from_iter(Some(id)); - let to_crop_ids = HashMap::new(); + let ids_in_formatted = vec![title, author]; + let mut formatted_options = HashMap::new(); + formatted_options.insert(title, FormatOptions { highlight: true, crop: None }); - let matching_words = HashSet::from_iter(Some(String::from("hello"))); + let matching_words = HashSet::from_iter(Some(String::from("hobbit"))); let value = compute_formatted( &fields, obkv, &formatter, &matching_words, - &all_formatted, - &to_highlight_ids, - &to_crop_ids, + &ids_in_formatted, + &formatted_options, ) .unwrap(); - assert_eq!(value["test"], "hello"); + assert_eq!(value["title"], "The Hobbit"); + assert_eq!(value["author"], "J. R. R. Tolkien"); + } + + #[test] + fn formatted_with_crop_2() { + let stop_words = fst::Set::default(); + let formatter = + Formatter::new(&stop_words, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let ids_in_formatted = vec![title, author]; + let mut formatted_options = HashMap::new(); + formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(2) }); + + let matching_words = HashSet::from_iter(Some(String::from("potter"))); + + let value = compute_formatted( + &fields, + obkv, + &formatter, + &matching_words, + &ids_in_formatted, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Harry Potter and"); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn formatted_with_crop_10() { + let stop_words = fst::Set::default(); + let formatter = + Formatter::new(&stop_words, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let ids_in_formatted = vec![title, author]; + let mut formatted_options = HashMap::new(); + formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(10) }); + + let matching_words = HashSet::from_iter(Some(String::from("potter"))); + + let value = compute_formatted( + &fields, + obkv, + &formatter, + &matching_words, + &ids_in_formatted, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Harry Potter and the Half"); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn formatted_with_crop_0() { + let stop_words = fst::Set::default(); + let formatter = + Formatter::new(&stop_words, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let ids_in_formatted = vec![title, author]; + let mut formatted_options = HashMap::new(); + formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(0) }); + + let matching_words = HashSet::from_iter(Some(String::from("potter"))); + + let value = compute_formatted( + &fields, + obkv, + &formatter, + &matching_words, + &ids_in_formatted, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Potter"); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn formatted_with_crop_and_highlight() { + let stop_words = fst::Set::default(); + let formatter = + Formatter::new(&stop_words, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let ids_in_formatted = vec![title, author]; + let mut formatted_options = HashMap::new(); + formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(1) }); + + let matching_words = HashSet::from_iter(Some(String::from("and"))); + + let value = compute_formatted( + &fields, + obkv, + &formatter, + &matching_words, + &ids_in_formatted, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], " and "); + assert_eq!(value["author"], "J. K. Rowling"); } } From 446b66b0fed92769606dc841da4f22921466a3cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 14 Jun 2021 14:59:38 +0200 Subject: [PATCH 410/522] Fix cargo clippy error --- meilisearch-http/src/index/search.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index b2c606c02..56fa5425f 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -182,7 +182,7 @@ impl Index { highlight = f.highlight; } formatted_options.insert(id, FormatOptions { - highlight: highlight, + highlight, crop: attr_len, }); } @@ -195,7 +195,7 @@ impl Index { highlight = f.highlight; } formatted_options.insert(id, FormatOptions { - highlight: highlight, + highlight, crop: attr_len, }); } @@ -362,14 +362,14 @@ fn compute_formatted>( obkv: obkv::KvReader, formatter: &Formatter, matching_words: &impl Matcher, - ids_in_formatted: &Vec, + ids_in_formatted: &[FieldId], formatted_options: &HashMap, // to_highlight_fields: &HashSet, //ICI // to_crop_fields: &HashMap>, //ICI ) -> anyhow::Result { let mut document = Document::new(); - if formatted_options.len() > 0 { + if !formatted_options.is_empty() { for field in ids_in_formatted { if let Some(value) = obkv.get(*field) { let mut value: Value = serde_json::from_slice(value)?; From b7698771833882dadd050813d4c1cbe8810112b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 14 Jun 2021 18:26:47 +0200 Subject: [PATCH 411/522] Make it compatible with the new milli highlighting --- meilisearch-http/src/index/search.rs | 131 ++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 15 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 56fa5425f..1d834234b 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -402,19 +402,25 @@ fn compute_formatted>( /// trait to allow unit testing of `compute_formatted` trait Matcher { - fn matches(&self, w: &str) -> bool; + fn matches(&self, w: &str) -> Option; } +// #[cfg(test)] +// impl Matcher for HashSet { +// fn matches(&self, w: &str) -> bool { +// self.contains(w) +// } +// } #[cfg(test)] -impl Matcher for HashSet { - fn matches(&self, w: &str) -> bool { - self.contains(w) +impl Matcher for HashMap<&str, Option> { + fn matches(&self, w: &str) -> Option { + self.get(w).cloned().flatten() } } impl Matcher for MatchingWords { - fn matches(&self, w: &str) -> bool { - self.matching_bytes(w).is_some() + fn matches(&self, w: &str) -> Option { + self.matching_bytes(w) } } @@ -476,7 +482,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { let mut buffer = VecDeque::new(); let mut tokens = analyzed.reconstruct().peekable(); let mut taken_before = 0; - while let Some((word, token)) = tokens.next_if(|(_, token)| !matcher.matches(token.text())) { + while let Some((word, token)) = tokens.next_if(|(_, token)| !matcher.matches(token.text()).is_some()) { buffer.push_back((word, token)); taken_before += word.chars().count(); while taken_before > crop_len { @@ -515,11 +521,14 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { tokens .map(|(word, token)| { - if need_to_highlight && token.is_word() && matcher.matches(token.text()) { + if need_to_highlight && token.is_word() && matcher.matches(token.text()).is_some() { let mut new_word = String::new(); new_word.push_str(&self.marks.0); - new_word.push_str(&word); - new_word.push_str(&self.marks.1); + if let Some(match_len) = matcher.matches(token.text()) { + new_word.push_str(&word[..match_len]); + new_word.push_str(&self.marks.1); + new_word.push_str(&word[match_len..]); + } new_word } else { word.to_string() @@ -672,7 +681,9 @@ mod test { let mut formatted_options = HashMap::new(); formatted_options.insert(title, FormatOptions { highlight: true, crop: None }); - let matching_words = HashSet::from_iter(Some(String::from("hobbit"))); + // let matching_words = HashSet::from_iter(Some(String::from("hobbit"))); + let mut matching_words = HashMap::new(); + matching_words.insert("hobbit", Some(6)); let value = compute_formatted( &fields, @@ -688,6 +699,49 @@ mod test { assert_eq!(value["author"], "J. R. R. Tolkien"); } + #[test] + fn formatted_with_highlight_in_word() { + let stop_words = fst::Set::default(); + let formatter = + Formatter::new(&stop_words, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("The Hobbit".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(author, Value::String("J. R. R. Tolkien".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let ids_in_formatted = vec![title, author]; + let mut formatted_options = HashMap::new(); + formatted_options.insert(title, FormatOptions { highlight: true, crop: None }); + + let mut matching_words = HashMap::new(); + matching_words.insert("hobbit", Some(3)); + + let value = compute_formatted( + &fields, + obkv, + &formatter, + &matching_words, + &ids_in_formatted, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "The Hobbit"); + assert_eq!(value["author"], "J. R. R. Tolkien"); + } + #[test] fn formatted_with_crop_2() { let stop_words = fst::Set::default(); @@ -714,7 +768,8 @@ mod test { let mut formatted_options = HashMap::new(); formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(2) }); - let matching_words = HashSet::from_iter(Some(String::from("potter"))); + let mut matching_words = HashMap::new(); + matching_words.insert("potter", Some(6)); let value = compute_formatted( &fields, @@ -756,7 +811,8 @@ mod test { let mut formatted_options = HashMap::new(); formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(10) }); - let matching_words = HashSet::from_iter(Some(String::from("potter"))); + let mut matching_words = HashMap::new(); + matching_words.insert("potter", Some(6)); let value = compute_formatted( &fields, @@ -798,7 +854,8 @@ mod test { let mut formatted_options = HashMap::new(); formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(0) }); - let matching_words = HashSet::from_iter(Some(String::from("potter"))); + let mut matching_words = HashMap::new(); + matching_words.insert("potter", Some(6)); let value = compute_formatted( &fields, @@ -840,7 +897,8 @@ mod test { let mut formatted_options = HashMap::new(); formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(1) }); - let matching_words = HashSet::from_iter(Some(String::from("and"))); + let mut matching_words = HashMap::new(); + matching_words.insert("and", Some(3)); let value = compute_formatted( &fields, @@ -855,4 +913,47 @@ mod test { assert_eq!(value["title"], " and "); assert_eq!(value["author"], "J. K. Rowling"); } + + #[test] + fn formatted_with_crop_and_highlight_in_word() { + let stop_words = fst::Set::default(); + let formatter = + Formatter::new(&stop_words, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let ids_in_formatted = vec![title, author]; + let mut formatted_options = HashMap::new(); + formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(9) }); + + let mut matching_words = HashMap::new(); + matching_words.insert("blood", Some(3)); + + let value = compute_formatted( + &fields, + obkv, + &formatter, + &matching_words, + &ids_in_formatted, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "the Half-Blood Prince"); + assert_eq!(value["author"], "J. K. Rowling"); + } } From 5dffe566fd59a5cdc6d40df9f8dc0b5328cc2a97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 14 Jun 2021 18:40:15 +0200 Subject: [PATCH 412/522] Remove useless comments --- meilisearch-http/src/index/search.rs | 86 +--------------------------- 1 file changed, 1 insertion(+), 85 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 1d834234b..37aea6248 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,4 +1,3 @@ -// use std::borrow::Cow; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::time::Instant; @@ -207,82 +206,13 @@ impl Index { .cloned() .collect::>(); - // All attributes present in `_formatted` that are not necessary highighted or croped + // All attributes present in `_formatted` that are not necessary highighted or cropped let ids_in_formatted = formatted_ids .union(&to_retrieve_ids) .cloned() .sorted() .collect::>(); - - // let to_highlight_ids = query // PLUS BESOIN - // .attributes_to_highlight - // .as_ref() - // .map(fids) - // .unwrap_or_default(); - - - // let to_crop_ids_length = query - // .attributes_to_crop - // .as_ref() - // .map(|attributes: &Vec| { - // let mut ids_length_crop = HashMap::new(); - // for attribute in attributes { - // let mut attr_name = attribute.clone(); - // let mut attr_len = Some(query.crop_length); - - // if attr_name.contains(':') { - // let mut split = attr_name.rsplit(':'); - // attr_len = match split.next() { - // Some(s) => s.parse::().ok(), - // None => None, - // }; - // attr_name = split.flat_map(|s| s.chars()).collect(); - // } - - // if attr_name == "*" { - // let ids = displayed_ids.clone(); - // for id in ids { - // ids_length_crop.insert(id, attr_len); - // } - // } - - // if let Some(id) = fields_ids_map.id(&attr_name) { - // if displayed_ids.contains(&id) { - // ids_length_crop.insert(id, attr_len); - // } - // } - // } - // ids_length_crop - // }) - // .unwrap_or_default(); - - // let to_crop_ids = to_crop_ids_length // PLUS BESOIN - // .clone() - // .into_iter() - // .map(|(k, _)| k) - // .collect::>(); - - // The formatted attributes are: - // - The one in either highlighted attributes or cropped attributes if there are attributes - // to retrieve - // - All the attributes to retrieve if there are either highlighted or cropped attributes - // the request specified that all attributes are to retrieve (i.e attributes to retrieve is - // empty in the query) - // let all_formatted = if query.attributes_to_retrieve.is_none() { - // if query.attributes_to_highlight.is_some() || query.attributes_to_crop.is_some() { - // Cow::Borrowed(&all_attributes) - // } else { - // Cow::Owned(Vec::new()) - // } - // } else { - // let attrs = (&to_crop_ids | &to_highlight_ids) - // .intersection(&displayed_ids) - // .cloned() - // .collect::>(); - // Cow::Owned(attrs) - // }; - let stop_words = fst::Set::default(); let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); @@ -295,10 +225,7 @@ impl Index { &formatter, &matching_words, &ids_in_formatted, - // all_formatted.as_ref().as_slice(), &formatted_options, - // &to_highlight_ids, //ICI - // &to_crop_ids_length, //ICI )?; let hit = SearchHit { document, @@ -364,8 +291,6 @@ fn compute_formatted>( matching_words: &impl Matcher, ids_in_formatted: &[FieldId], formatted_options: &HashMap, - // to_highlight_fields: &HashSet, //ICI - // to_crop_fields: &HashMap>, //ICI ) -> anyhow::Result { let mut document = Document::new(); @@ -380,8 +305,6 @@ fn compute_formatted>( matching_words, format.highlight, format.crop, - // to_crop_fields.get(field).copied().flatten(), //ICI - // to_highlight_fields.contains(field), //ICI ); } @@ -405,12 +328,6 @@ trait Matcher { fn matches(&self, w: &str) -> Option; } -// #[cfg(test)] -// impl Matcher for HashSet { -// fn matches(&self, w: &str) -> bool { -// self.contains(w) -// } -// } #[cfg(test)] impl Matcher for HashMap<&str, Option> { fn matches(&self, w: &str) -> Option { @@ -681,7 +598,6 @@ mod test { let mut formatted_options = HashMap::new(); formatted_options.insert(title, FormatOptions { highlight: true, crop: None }); - // let matching_words = HashSet::from_iter(Some(String::from("hobbit"))); let mut matching_words = HashMap::new(); matching_words.insert("hobbit", Some(6)); From d9c0190497ec2f6320991c9010e6bead07c274dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 15 Jun 2021 16:21:41 +0200 Subject: [PATCH 413/522] Redo to_retrieve_ids --- meilisearch-http/src/index/search.rs | 45 ++++++++++++++++------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 37aea6248..5e4ee6e6f 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -113,31 +113,37 @@ impl Index { } if let Some(id) = fields_ids_map.id(attr) { - if displayed_ids.contains(&id) { - ids.insert(id); - } + ids.insert(id); } } ids }; - let to_retrieve_ids = query + // The attributes to retrieve are the ones explicitly marked as to retrieve (all by default), + // but these attributes must be also + // - present in the fields_ids_map + // - present in the the displayed attributes + let to_retrieve_ids: HashSet<_> = query .attributes_to_retrieve .as_ref() .map(fids) - .unwrap_or_else(|| displayed_ids.clone()); - - // The attributes to retrieve are: - // - the ones explicitly marked as to retrieve that are also in the displayed attributes - let all_attributes: Vec<_> = to_retrieve_ids + .unwrap_or_else(|| displayed_ids.clone()) .intersection(&displayed_ids) .cloned() + .collect(); + + let to_retrieve_ids_sorted: Vec<_> = to_retrieve_ids + .clone() + .into_iter() .sorted() .collect(); let mut formatted_options = HashMap::new(); - let attr_to_highlight = query.attributes_to_highlight.unwrap_or_default(); + let attr_to_highlight = query + .attributes_to_highlight + .unwrap_or_default(); + for attr in attr_to_highlight { let new_format = FormatOptions { highlight: true, @@ -159,7 +165,10 @@ impl Index { } }; - let attr_to_crop = query.attributes_to_crop.unwrap_or_default(); + let attr_to_crop = query + .attributes_to_crop + .unwrap_or_default(); + for attr in attr_to_crop { let mut attr_name = attr.clone(); let mut attr_len = Some(query.crop_length); @@ -201,13 +210,13 @@ impl Index { } } - let formatted_ids = formatted_options + // All attributes present in `_formatted`: + // - attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`) + // - attributes asked to be retrieved: these attributes will not be formatted + let ids_in_formatted = formatted_options .keys() .cloned() - .collect::>(); - - // All attributes present in `_formatted` that are not necessary highighted or cropped - let ids_in_formatted = formatted_ids + .collect::>() .union(&to_retrieve_ids) .cloned() .sorted() @@ -218,7 +227,7 @@ impl Index { Formatter::new(&stop_words, (String::from(""), String::from(""))); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let document = make_document(&all_attributes, &fields_ids_map, obkv)?; + let document = make_document(&to_retrieve_ids_sorted, &fields_ids_map, obkv)?; let formatted = compute_formatted( &fields_ids_map, obkv, @@ -498,8 +507,6 @@ fn parse_facets_array( #[cfg(test)] mod test { - use std::iter::FromIterator; - use super::*; #[test] From 5e656bb58a452d42a3148d530c7ca8dee580344b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 15 Jun 2021 16:25:16 +0200 Subject: [PATCH 414/522] Rename parse_facets into parse_filter --- meilisearch-http/src/index/search.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 5e4ee6e6f..4b178815f 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -85,7 +85,7 @@ impl Index { search.offset(query.offset.unwrap_or_default()); if let Some(ref filter) = query.filter { - if let Some(facets) = parse_facets(filter, self, &rtxn)? { + if let Some(facets) = parse_filter(filter, self, &rtxn)? { search.filter(facets); } } @@ -464,19 +464,19 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { } } -fn parse_facets( +fn parse_filter( facets: &Value, index: &Index, txn: &RoTxn, ) -> anyhow::Result> { match facets { Value::String(expr) => Ok(Some(FilterCondition::from_str(txn, index, expr)?)), - Value::Array(arr) => parse_facets_array(txn, index, arr), + Value::Array(arr) => parse_filter_array(txn, index, arr), v => bail!("Invalid facet expression, expected Array, found: {:?}", v), } } -fn parse_facets_array( +fn parse_filter_array( txn: &RoTxn, index: &Index, arr: &[Value], From 8d0269fcc4386a0022ea50954d4d1172fb4c562b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 15 Jun 2021 17:16:07 +0200 Subject: [PATCH 415/522] Create function to create fomatted_options --- meilisearch-http/src/index/search.rs | 146 +++++++++++++++------------ 1 file changed, 83 insertions(+), 63 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 4b178815f..e4c7777a0 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -138,81 +138,26 @@ impl Index { .sorted() .collect(); - let mut formatted_options = HashMap::new(); - let attr_to_highlight = query .attributes_to_highlight .unwrap_or_default(); - for attr in attr_to_highlight { - let new_format = FormatOptions { - highlight: true, - crop: None, - }; - - if attr == "*" { - let ids = displayed_ids.clone(); - for id in ids { - formatted_options.insert(id, new_format); - } - break; - } - - if let Some(id) = fields_ids_map.id(&attr) { - if displayed_ids.contains(&id) { - formatted_options.insert(id, new_format); - } - } - }; - let attr_to_crop = query .attributes_to_crop .unwrap_or_default(); - for attr in attr_to_crop { - let mut attr_name = attr.clone(); - let mut attr_len = Some(query.crop_length); - - if attr_name.contains(':') { - let mut split = attr_name.rsplit(':'); - attr_len = match split.next() { - Some(s) => s.parse::().ok(), - None => None, - }; - attr_name = split.flat_map(|s| s.chars()).collect(); - } - - if attr_name == "*" { - let ids = displayed_ids.clone(); - for id in ids { - let mut highlight = false; - if let Some(f) = formatted_options.get(&id) { - highlight = f.highlight; - } - formatted_options.insert(id, FormatOptions { - highlight, - crop: attr_len, - }); - } - } - - if let Some(id) = fields_ids_map.id(&attr_name) { - if displayed_ids.contains(&id) { - let mut highlight = false; - if let Some(f) = formatted_options.get(&id) { - highlight = f.highlight; - } - formatted_options.insert(id, FormatOptions { - highlight, - crop: attr_len, - }); - } - } - } + let formatted_options = parse_formatted_options( + &attr_to_highlight, + &attr_to_crop, + query.crop_length, + &fields_ids_map, + &displayed_ids, + ); // All attributes present in `_formatted`: // - attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`) // - attributes asked to be retrieved: these attributes will not be formatted + // - attributes that are present in displayed attributes let ids_in_formatted = formatted_options .keys() .cloned() @@ -270,6 +215,81 @@ impl Index { } } +fn parse_formatted_options( + attr_to_highlight: &HashSet, + attr_to_crop: &[String], + query_crop_length: usize, + fields_ids_map: &FieldsIdsMap, + displayed_ids: &HashSet, + ) -> HashMap { + + let mut formatted_options = HashMap::new(); + + for attr in attr_to_highlight { + let new_format = FormatOptions { + highlight: true, + crop: None, + }; + + if attr == "*" { + let ids = displayed_ids.clone(); + for id in ids { + formatted_options.insert(id, new_format); + } + break; + } + + if let Some(id) = fields_ids_map.id(&attr) { + if displayed_ids.contains(&id) { + formatted_options.insert(id, new_format); + } + } + }; + + for attr in attr_to_crop { + let mut attr_name = attr.clone(); + let mut attr_len = Some(query_crop_length); + + if attr_name.contains(':') { + let mut split = attr_name.rsplit(':'); + attr_len = match split.next() { + Some(s) => s.parse::().ok(), + None => None, + }; + attr_name = split.flat_map(|s| s.chars()).collect(); + } + + if attr_name == "*" { + let ids = displayed_ids.clone(); + for id in ids { + let mut highlight = false; + if let Some(f) = formatted_options.get(&id) { + highlight = f.highlight; + } + formatted_options.insert(id, FormatOptions { + highlight, + crop: attr_len, + }); + } + } + + if let Some(id) = fields_ids_map.id(&attr_name) { + if displayed_ids.contains(&id) { + let mut highlight = false; + if let Some(f) = formatted_options.get(&id) { + highlight = f.highlight; + } + formatted_options.insert(id, FormatOptions { + highlight, + crop: attr_len, + }); + } + } + } + + formatted_options +} + fn make_document( attributes_to_retrieve: &[FieldId], field_ids_map: &FieldsIdsMap, From 79a1212ebea46403389cfc4b0e632a919f5161a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 15 Jun 2021 17:28:26 +0200 Subject: [PATCH 416/522] Do intersection with displayed ids instead of checking in loop --- meilisearch-http/src/index/search.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index e4c7777a0..c1275d089 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -162,6 +162,9 @@ impl Index { .keys() .cloned() .collect::>() + .intersection(&displayed_ids) + .cloned() + .collect::>() .union(&to_retrieve_ids) .cloned() .sorted() @@ -240,9 +243,7 @@ fn parse_formatted_options( } if let Some(id) = fields_ids_map.id(&attr) { - if displayed_ids.contains(&id) { - formatted_options.insert(id, new_format); - } + formatted_options.insert(id, new_format); } }; @@ -274,16 +275,14 @@ fn parse_formatted_options( } if let Some(id) = fields_ids_map.id(&attr_name) { - if displayed_ids.contains(&id) { - let mut highlight = false; - if let Some(f) = formatted_options.get(&id) { - highlight = f.highlight; - } - formatted_options.insert(id, FormatOptions { - highlight, - crop: attr_len, - }); + let mut highlight = false; + if let Some(f) = formatted_options.get(&id) { + highlight = f.highlight; } + formatted_options.insert(id, FormatOptions { + highlight, + crop: attr_len, + }); } } From 1ef061d92bfe53c1a69ad2ba589430b0a937434c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 15 Jun 2021 17:31:32 +0200 Subject: [PATCH 417/522] Fix clippy errors --- meilisearch-http/src/index/search.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index c1275d089..cfb2f77d1 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -154,10 +154,10 @@ impl Index { &displayed_ids, ); - // All attributes present in `_formatted`: - // - attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`) - // - attributes asked to be retrieved: these attributes will not be formatted - // - attributes that are present in displayed attributes + // All attributes present in `_formatted` are: + // - the attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`) + // - the attributes asked to be retrieved: these attributes will not be highlighted/cropped + // But these attributes must be present in displayed attributes let ids_in_formatted = formatted_options .keys() .cloned() @@ -427,7 +427,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { let mut buffer = VecDeque::new(); let mut tokens = analyzed.reconstruct().peekable(); let mut taken_before = 0; - while let Some((word, token)) = tokens.next_if(|(_, token)| !matcher.matches(token.text()).is_some()) { + while let Some((word, token)) = tokens.next_if(|(_, token)| matcher.matches(token.text()).is_none()) { buffer.push_back((word, token)); taken_before += word.chars().count(); while taken_before > crop_len { From 9840b5c7fb59a5afa773bc696aa711fb50818cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 15 Jun 2021 18:44:56 +0200 Subject: [PATCH 418/522] Refacto --- meilisearch-http/src/index/search.rs | 58 +++++++++++++--------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index cfb2f77d1..c2e57af75 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -235,9 +235,8 @@ fn parse_formatted_options( }; if attr == "*" { - let ids = displayed_ids.clone(); - for id in ids { - formatted_options.insert(id, new_format); + for id in displayed_ids { + formatted_options.insert(*id, new_format); } break; } @@ -245,44 +244,41 @@ fn parse_formatted_options( if let Some(id) = fields_ids_map.id(&attr) { formatted_options.insert(id, new_format); } - }; + } for attr in attr_to_crop { let mut attr_name = attr.clone(); - let mut attr_len = Some(query_crop_length); + let mut attr_len = query_crop_length; - if attr_name.contains(':') { - let mut split = attr_name.rsplit(':'); - attr_len = match split.next() { - Some(s) => s.parse::().ok(), - None => None, - }; - attr_name = split.flat_map(|s| s.chars()).collect(); - } + let mut split = attr_name.rsplitn(2, ':'); + attr_name = match split.next().zip(split.next()) { + Some((len, name)) => { + attr_len = len.parse().unwrap_or(query_crop_length); + name.to_string() + }, + None => attr_name, + }; if attr_name == "*" { - let ids = displayed_ids.clone(); - for id in ids { - let mut highlight = false; - if let Some(f) = formatted_options.get(&id) { - highlight = f.highlight; - } - formatted_options.insert(id, FormatOptions { - highlight, - crop: attr_len, - }); + for id in displayed_ids { + formatted_options + .entry(*id) + .and_modify(|f| f.crop = Some(attr_len)) + .or_insert(FormatOptions { + highlight: false, + crop: Some(attr_len), + }); } } if let Some(id) = fields_ids_map.id(&attr_name) { - let mut highlight = false; - if let Some(f) = formatted_options.get(&id) { - highlight = f.highlight; - } - formatted_options.insert(id, FormatOptions { - highlight, - crop: attr_len, - }); + formatted_options + .entry(id) + .and_modify(|f| f.crop = Some(attr_len)) + .or_insert(FormatOptions { + highlight: false, + crop: Some(attr_len), + }); } } From c0d169e79e765088251fd63f7bf6cb975c2d05c4 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Wed, 16 Jun 2021 11:12:46 +0200 Subject: [PATCH 419/522] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clémentine Urquizar --- meilisearch-http/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 120684dfc..ae2cb3777 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -129,10 +129,10 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { eprintln!( " Thank you for using MeiliSearch! + We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics -Anonymous telemetry: \"Enabled\" -" +Anonymous telemetry: \"Enabled\"" ); } } From 7b02fdaddc64870f935ff1a033152dad931d29ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 16 Jun 2021 14:23:08 +0200 Subject: [PATCH 420/522] Rename functions --- meilisearch-http/src/index/search.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index c2e57af75..b5ba86a16 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -146,7 +146,7 @@ impl Index { .attributes_to_crop .unwrap_or_default(); - let formatted_options = parse_formatted_options( + let formatted_options = compute_formatted_options( &attr_to_highlight, &attr_to_crop, query.crop_length, @@ -176,7 +176,7 @@ impl Index { for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let document = make_document(&to_retrieve_ids_sorted, &fields_ids_map, obkv)?; - let formatted = compute_formatted( + let formatted = format_fields( &fields_ids_map, obkv, &formatter, @@ -218,7 +218,7 @@ impl Index { } } -fn parse_formatted_options( +fn compute_formatted_options( attr_to_highlight: &HashSet, attr_to_crop: &[String], query_crop_length: usize, @@ -308,7 +308,7 @@ fn make_document( Ok(document) } -fn compute_formatted>( +fn format_fields>( field_ids_map: &FieldsIdsMap, obkv: obkv::KvReader, formatter: &Formatter, @@ -347,7 +347,7 @@ fn compute_formatted>( Ok(document) } -/// trait to allow unit testing of `compute_formatted` +/// trait to allow unit testing of `format_fields` trait Matcher { fn matches(&self, w: &str) -> Option; } @@ -546,7 +546,7 @@ mod test { let matching_words = MatchingWords::default(); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, @@ -581,7 +581,7 @@ mod test { let matching_words = MatchingWords::default(); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, @@ -623,7 +623,7 @@ mod test { let mut matching_words = HashMap::new(); matching_words.insert("hobbit", Some(6)); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, @@ -666,7 +666,7 @@ mod test { let mut matching_words = HashMap::new(); matching_words.insert("hobbit", Some(3)); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, @@ -709,7 +709,7 @@ mod test { let mut matching_words = HashMap::new(); matching_words.insert("potter", Some(6)); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, @@ -752,7 +752,7 @@ mod test { let mut matching_words = HashMap::new(); matching_words.insert("potter", Some(6)); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, @@ -795,7 +795,7 @@ mod test { let mut matching_words = HashMap::new(); matching_words.insert("potter", Some(6)); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, @@ -838,7 +838,7 @@ mod test { let mut matching_words = HashMap::new(); matching_words.insert("and", Some(3)); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, @@ -881,7 +881,7 @@ mod test { let mut matching_words = HashMap::new(); matching_words.insert("blood", Some(3)); - let value = compute_formatted( + let value = format_fields( &fields, obkv, &formatter, From dc5a3d4a620a1fdf5b903a3b42ffe1be292575b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 16 Jun 2021 16:18:55 +0200 Subject: [PATCH 421/522] Use BTreeSet instead of HashSet --- meilisearch-http/src/index/search.rs | 49 +++++++++++---------------- meilisearch-http/src/routes/search.rs | 4 +-- 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index b5ba86a16..bc9d5ac47 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,11 +1,10 @@ -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::time::Instant; use anyhow::bail; use either::Either; use heed::RoTxn; use indexmap::IndexMap; -use itertools::Itertools; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token}; use milli::{FilterCondition, FieldId, FieldsIdsMap, MatchingWords}; use serde::{Deserialize, Serialize}; @@ -32,7 +31,7 @@ pub struct SearchQuery { pub offset: Option, #[serde(default = "default_search_limit")] pub limit: usize, - pub attributes_to_retrieve: Option>, + pub attributes_to_retrieve: Option>, pub attributes_to_crop: Option>, #[serde(default = "default_crop_length")] pub crop_length: usize, @@ -101,11 +100,11 @@ impl Index { let displayed_ids = self .displayed_fields_ids(&rtxn)? - .map(|fields| fields.into_iter().collect::>()) + .map(|fields| fields.into_iter().collect::>()) .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect()); - let fids = |attrs: &HashSet| { - let mut ids = HashSet::new(); + let fids = |attrs: &BTreeSet| { + let mut ids = BTreeSet::new(); for attr in attrs { if attr == "*" { ids = displayed_ids.clone(); @@ -123,7 +122,7 @@ impl Index { // but these attributes must be also // - present in the fields_ids_map // - present in the the displayed attributes - let to_retrieve_ids: HashSet<_> = query + let to_retrieve_ids: BTreeSet<_> = query .attributes_to_retrieve .as_ref() .map(fids) @@ -132,12 +131,6 @@ impl Index { .cloned() .collect(); - let to_retrieve_ids_sorted: Vec<_> = to_retrieve_ids - .clone() - .into_iter() - .sorted() - .collect(); - let attr_to_highlight = query .attributes_to_highlight .unwrap_or_default(); @@ -161,13 +154,12 @@ impl Index { let ids_in_formatted = formatted_options .keys() .cloned() - .collect::>() + .collect::>() .intersection(&displayed_ids) .cloned() - .collect::>() + .collect::>() .union(&to_retrieve_ids) .cloned() - .sorted() .collect::>(); let stop_words = fst::Set::default(); @@ -175,7 +167,7 @@ impl Index { Formatter::new(&stop_words, (String::from(""), String::from(""))); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let document = make_document(&to_retrieve_ids_sorted, &fields_ids_map, obkv)?; + let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?; let formatted = format_fields( &fields_ids_map, obkv, @@ -223,7 +215,7 @@ fn compute_formatted_options( attr_to_crop: &[String], query_crop_length: usize, fields_ids_map: &FieldsIdsMap, - displayed_ids: &HashSet, + displayed_ids: &BTreeSet, ) -> HashMap { let mut formatted_options = HashMap::new(); @@ -286,7 +278,7 @@ fn compute_formatted_options( } fn make_document( - attributes_to_retrieve: &[FieldId], + attributes_to_retrieve: &BTreeSet, field_ids_map: &FieldsIdsMap, obkv: obkv::KvReader, ) -> anyhow::Result { @@ -327,8 +319,7 @@ fn format_fields>( value = formatter.format_value( value, matching_words, - format.highlight, - format.crop, + *format, ); } @@ -384,25 +375,24 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { &self, value: Value, matcher: &impl Matcher, - need_to_highlight: bool, - need_to_crop: Option, + format_options: FormatOptions, ) -> Value { match value { Value::String(old_string) => { let value = - self.format_string(old_string, matcher, need_to_highlight, need_to_crop); + self.format_string(old_string, matcher, format_options); Value::String(value) } Value::Array(values) => Value::Array( values .into_iter() - .map(|v| self.format_value(v, matcher, need_to_highlight, None)) + .map(|v| self.format_value(v, matcher, FormatOptions { highlight: format_options.highlight, crop: None })) .collect(), ), Value::Object(object) => Value::Object( object .into_iter() - .map(|(k, v)| (k, self.format_value(v, matcher, need_to_highlight, None))) + .map(|(k, v)| (k, self.format_value(v, matcher, FormatOptions { highlight: format_options.highlight, crop: None }))) .collect(), ), value => value, @@ -413,12 +403,11 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { &self, s: String, matcher: &impl Matcher, - need_to_highlight: bool, - need_to_crop: Option, + format_options: FormatOptions, ) -> String { let analyzed = self.analyzer.analyze(&s); - let tokens: Box> = match need_to_crop { + let tokens: Box> = match format_options.crop { Some(crop_len) => { let mut buffer = VecDeque::new(); let mut tokens = analyzed.reconstruct().peekable(); @@ -462,7 +451,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { tokens .map(|(word, token)| { - if need_to_highlight && token.is_word() && matcher.matches(token.text()).is_some() { + if format_options.highlight && token.is_word() && matcher.matches(token.text()).is_some() { let mut new_word = String::new(); new_word.push_str(&self.marks.0); if let Some(match_len) = matcher.matches(token.text()) { diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index c2c83e3c8..36f5bdf4d 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; use std::convert::{TryFrom, TryInto}; use actix_web::{get, post, web, HttpResponse}; @@ -36,7 +36,7 @@ impl TryFrom for SearchQuery { fn try_from(other: SearchQueryGet) -> anyhow::Result { let attributes_to_retrieve = other .attributes_to_retrieve - .map(|attrs| attrs.split(',').map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let attributes_to_crop = other .attributes_to_crop From 4e2568fd6e18f048c84cb5c29612f0fffee41f17 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 16 Jun 2021 17:12:49 +0200 Subject: [PATCH 422/522] disable amplitude on debug build --- meilisearch-http/src/lib.rs | 2 +- meilisearch-http/src/main.rs | 6 +++--- meilisearch-http/src/option.rs | 2 +- meilisearch-http/tests/common/server.rs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 8788d45d1..130389fdc 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -6,7 +6,7 @@ mod index_controller; pub mod option; pub mod routes; -#[cfg(feature = "analytics")] +#[cfg(all(not(debug_assertions), feature = "analytics"))] pub mod analytics; pub use self::data::Data; diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index ae2cb3777..2787d8b78 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -5,7 +5,7 @@ use main_error::MainError; use meilisearch_http::{create_app, Data, Opt}; use structopt::StructOpt; -#[cfg(feature = "analytics")] +#[cfg(all(not(debug_assertions), feature = "analytics"))] use meilisearch_http::analytics; #[cfg(target_os = "linux")] @@ -57,7 +57,7 @@ async fn main() -> Result<(), MainError> { let data = Data::new(opt.clone())?; - #[cfg(feature = "analytics")] + #[cfg(all(not(debug_assertions), feature = "analytics"))] if !opt.no_analytics { let analytics_data = data.clone(); let analytics_opt = opt.clone(); @@ -121,7 +121,7 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { env!("CARGO_PKG_VERSION").to_string() ); - #[cfg(feature = "analytics")] + #[cfg(all(not(debug_assertions), feature = "analytics"))] { if opt.no_analytics { eprintln!("Anonymous telemetry:\t\"Disabled\""); diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 54a50742a..8f925bad8 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -104,7 +104,7 @@ pub struct Opt { pub env: String, /// Do not send analytics to Meili. - #[cfg(feature = "analytics")] + #[cfg(all(not(debug_assertions), feature = "analytics"))] #[structopt(long, env = "MEILI_NO_ANALYTICS")] pub no_analytics: bool, diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 2c47a8ca8..bf6c95474 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -71,7 +71,7 @@ pub fn default_settings(dir: impl AsRef) -> Opt { http_addr: "127.0.0.1:7700".to_owned(), master_key: None, env: "development".to_owned(), - #[cfg(feature = "analytics")] + #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics: true, max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), From 9538790b33334f706145b2e35f0036fa0dca95cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 16 Jun 2021 17:13:21 +0200 Subject: [PATCH 423/522] Decompose into two functions --- meilisearch-http/src/index/search.rs | 37 ++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index bc9d5ac47..46e1e6e57 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -220,6 +220,30 @@ fn compute_formatted_options( let mut formatted_options = HashMap::new(); + formatted_options = add_highlight_to_formatted_options( + formatted_options, + attr_to_highlight, + fields_ids_map, + displayed_ids, + ); + + formatted_options = add_crop_to_formatted_options( + formatted_options, + attr_to_crop, + query_crop_length, + fields_ids_map, + displayed_ids, + ); + + formatted_options +} + +fn add_highlight_to_formatted_options( + mut formatted_options: HashMap, + attr_to_highlight: &HashSet, + fields_ids_map: &FieldsIdsMap, + displayed_ids: &BTreeSet, +) -> HashMap { for attr in attr_to_highlight { let new_format = FormatOptions { highlight: true, @@ -237,15 +261,24 @@ fn compute_formatted_options( formatted_options.insert(id, new_format); } } + formatted_options +} +fn add_crop_to_formatted_options( + mut formatted_options: HashMap, + attr_to_crop: &[String], + crop_length: usize, + fields_ids_map: &FieldsIdsMap, + displayed_ids: &BTreeSet, +) -> HashMap { for attr in attr_to_crop { let mut attr_name = attr.clone(); - let mut attr_len = query_crop_length; + let mut attr_len = crop_length; let mut split = attr_name.rsplitn(2, ':'); attr_name = match split.next().zip(split.next()) { Some((len, name)) => { - attr_len = len.parse().unwrap_or(query_crop_length); + attr_len = len.parse().unwrap_or(crop_length); name.to_string() }, None => attr_name, From 5a47cef9a88c7e2a8228198ae323f118bf36538e Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 16 Jun 2021 17:15:56 +0200 Subject: [PATCH 424/522] bump milli to 0.4.0 --- Cargo.lock | 5 ++--- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/index/search.rs | 2 +- meilisearch-http/src/index/updates.rs | 11 +++++++---- .../src/index_controller/index_actor/actor.rs | 11 ++++++++--- .../src/index_controller/index_actor/mod.rs | 8 ++++++-- .../src/index_controller/index_actor/store.rs | 8 +++++++- 7 files changed, 32 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f10741b55..3605f23e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1660,10 +1660,9 @@ dependencies = [ [[package]] name = "milli" -version = "0.3.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.3.1#bc020317935da4ea08061b3d4518cbbd40184856" +version = "0.4.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.4.0#3bd4cf94cc60733393b94021fca77eb100bfe17a" dependencies = [ - "anyhow", "bstr", "byteorder", "chrono", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 23ccbe42c..03037cb1b 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.3.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.4.0" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 50d163898..783c3d3e0 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -369,7 +369,7 @@ fn parse_facets_array( } } - FilterCondition::from_array(txn, &index.0, ands) + Ok(FilterCondition::from_array(txn, &index.0, ands)?) } #[cfg(test)] diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index fc475aa4d..ce327520e 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -200,8 +200,11 @@ impl Index { info!("performing document addition"); // Set the primary key if not set already, ignore if already set. - if let (None, Some(ref primary_key)) = (self.primary_key(txn)?, primary_key) { - self.put_primary_key(txn, primary_key)?; + if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) { + let mut builder = UpdateBuilder::new(0) + .settings(txn, &self); + builder.set_primary_key(primary_key.to_string()); + builder.execute(|_, _| ())?; } let mut builder = update_builder.index_documents(txn, self); @@ -235,7 +238,7 @@ impl Index { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e), + Err(e) => Err(e.into()), } } @@ -331,7 +334,7 @@ impl Index { .commit() .and(Ok(UpdateResult::DocumentDeletion { deleted })) .map_err(Into::into), - Err(e) => Err(e), + Err(e) => Err(e.into()), } } } diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 4d8324e11..c35e685a8 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -6,6 +6,7 @@ use async_stream::stream; use futures::stream::StreamExt; use heed::CompactionOption; use log::debug; +use milli::update::UpdateBuilder; use tokio::task::spawn_blocking; use tokio::{fs, sync::mpsc}; use uuid::Uuid; @@ -258,12 +259,15 @@ impl IndexActor { .ok_or(IndexError::UnexistingIndex)?; let result = spawn_blocking(move || match index_settings.primary_key { - Some(ref primary_key) => { + Some(primary_key) => { let mut txn = index.write_txn()?; if index.primary_key(&txn)?.is_some() { return Err(IndexError::ExistingPrimaryKey); } - index.put_primary_key(&mut txn, primary_key)?; + let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); + builder.set_primary_key(primary_key); + builder.execute(|_, _| ()) + .map_err(|e| IndexError::Internal(e.to_string()))?; let meta = IndexMeta::new_txn(&index, &txn)?; txn.commit()?; Ok(meta) @@ -333,7 +337,8 @@ impl IndexActor { Ok(IndexStats { size: index.size(), - number_of_documents: index.number_of_documents(&rtxn)?, + number_of_documents: index.number_of_documents(&rtxn) + .map_err(|e| IndexError::Internal(e.to_string()))?, is_indexing: None, fields_distribution: index.fields_distribution(&rtxn)?, }) diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 1ddc0199e..b54a676b0 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -41,8 +41,12 @@ impl IndexMeta { } fn new_txn(index: &Index, txn: &heed::RoTxn) -> IndexResult { - let created_at = index.created_at(&txn)?; - let updated_at = index.updated_at(&txn)?; + let created_at = index + .created_at(&txn) + .map_err(|e| IndexError::Internal(e.to_string()))?; + let updated_at = index + .updated_at(&txn) + .map_err(|e| IndexError::Internal(e.to_string()))?; let primary_key = index.primary_key(&txn)?.map(String::from); Ok(Self { created_at, diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 1646821d8..39a6e64a6 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; +use milli::update::UpdateBuilder; use tokio::fs; use tokio::sync::RwLock; use tokio::task::spawn_blocking; @@ -57,7 +58,12 @@ impl IndexStore for MapIndexStore { let index = Index::open(path, index_size)?; if let Some(primary_key) = primary_key { let mut txn = index.write_txn()?; - index.put_primary_key(&mut txn, &primary_key)?; + + let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); + builder.set_primary_key(primary_key); + builder.execute(|_, _| ()) + .map_err(|e| IndexError::Internal(e.to_string()))?; + txn.commit()?; } Ok(index) From a0b022afee3e5fd0398b06b39bb831334dd5ddfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 16 Jun 2021 17:25:02 +0200 Subject: [PATCH 425/522] Add Cow --- meilisearch-http/src/index/search.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 46e1e6e57..190d6b760 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::time::Instant; @@ -492,9 +493,9 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { new_word.push_str(&self.marks.1); new_word.push_str(&word[match_len..]); } - new_word + Cow::Owned(new_word) } else { - word.to_string() + Cow::Borrowed(word) } }) .collect::() From 97909ce56e8971027c5850a722583ca7e003779e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 16 Jun 2021 19:30:06 +0200 Subject: [PATCH 426/522] Use BTreeMap and remove ids_in_formatted --- meilisearch-http/src/index/search.rs | 251 ++++++++++----------------- 1 file changed, 90 insertions(+), 161 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 190d6b760..40ac1b1d9 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,5 +1,5 @@ use std::borrow::Cow; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque}; use std::time::Instant; use anyhow::bail; @@ -120,9 +120,9 @@ impl Index { }; // The attributes to retrieve are the ones explicitly marked as to retrieve (all by default), - // but these attributes must be also - // - present in the fields_ids_map - // - present in the the displayed attributes + // but these attributes must be also be present + // - in the fields_ids_map + // - in the the displayed attributes let to_retrieve_ids: BTreeSet<_> = query .attributes_to_retrieve .as_ref() @@ -140,29 +140,20 @@ impl Index { .attributes_to_crop .unwrap_or_default(); + // Attributes in `formatted_options` correspond to the attributes that will be in `_formatted` + // These attributes are: + // - the attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`) + // - the attributes asked to be retrieved: these attributes will not be highlighted/cropped + // But these attributes must be present in displayed attributes let formatted_options = compute_formatted_options( &attr_to_highlight, &attr_to_crop, query.crop_length, + &to_retrieve_ids, &fields_ids_map, &displayed_ids, ); - // All attributes present in `_formatted` are: - // - the attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`) - // - the attributes asked to be retrieved: these attributes will not be highlighted/cropped - // But these attributes must be present in displayed attributes - let ids_in_formatted = formatted_options - .keys() - .cloned() - .collect::>() - .intersection(&displayed_ids) - .cloned() - .collect::>() - .union(&to_retrieve_ids) - .cloned() - .collect::>(); - let stop_words = fst::Set::default(); let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); @@ -174,7 +165,6 @@ impl Index { obkv, &formatter, &matching_words, - &ids_in_formatted, &formatted_options, )?; let hit = SearchHit { @@ -215,11 +205,12 @@ fn compute_formatted_options( attr_to_highlight: &HashSet, attr_to_crop: &[String], query_crop_length: usize, + to_retrieve_ids: &BTreeSet, fields_ids_map: &FieldsIdsMap, displayed_ids: &BTreeSet, - ) -> HashMap { + ) -> BTreeMap { - let mut formatted_options = HashMap::new(); + let mut formatted_options = BTreeMap::new(); formatted_options = add_highlight_to_formatted_options( formatted_options, @@ -236,15 +227,23 @@ fn compute_formatted_options( displayed_ids, ); + // Should not return `_formatted` if no valid attributes to highlight/crop + if !formatted_options.is_empty() { + formatted_options = add_non_formatted_ids_to_formatted_options( + formatted_options, + to_retrieve_ids, + ); + } + formatted_options } fn add_highlight_to_formatted_options( - mut formatted_options: HashMap, + mut formatted_options: BTreeMap, attr_to_highlight: &HashSet, fields_ids_map: &FieldsIdsMap, displayed_ids: &BTreeSet, -) -> HashMap { +) -> BTreeMap { for attr in attr_to_highlight { let new_format = FormatOptions { highlight: true, @@ -259,19 +258,22 @@ fn add_highlight_to_formatted_options( } if let Some(id) = fields_ids_map.id(&attr) { - formatted_options.insert(id, new_format); + if displayed_ids.contains(&id) { + formatted_options.insert(id, new_format); + } } } + formatted_options } fn add_crop_to_formatted_options( - mut formatted_options: HashMap, + mut formatted_options: BTreeMap, attr_to_crop: &[String], crop_length: usize, fields_ids_map: &FieldsIdsMap, displayed_ids: &BTreeSet, -) -> HashMap { +) -> BTreeMap { for attr in attr_to_crop { let mut attr_name = attr.clone(); let mut attr_len = crop_length; @@ -298,19 +300,37 @@ fn add_crop_to_formatted_options( } if let Some(id) = fields_ids_map.id(&attr_name) { - formatted_options - .entry(id) - .and_modify(|f| f.crop = Some(attr_len)) - .or_insert(FormatOptions { - highlight: false, - crop: Some(attr_len), - }); + if displayed_ids.contains(&id) { + formatted_options + .entry(id) + .and_modify(|f| f.crop = Some(attr_len)) + .or_insert(FormatOptions { + highlight: false, + crop: Some(attr_len), + }); + } } } formatted_options } +fn add_non_formatted_ids_to_formatted_options( + mut formatted_options: BTreeMap, + to_retrieve_ids: &BTreeSet +) -> BTreeMap { + for id in to_retrieve_ids { + formatted_options + .entry(*id) + .or_insert(FormatOptions { + highlight: false, + crop: None, + }); + } + + formatted_options +} + fn make_document( attributes_to_retrieve: &BTreeSet, field_ids_map: &FieldsIdsMap, @@ -339,33 +359,28 @@ fn format_fields>( obkv: obkv::KvReader, formatter: &Formatter, matching_words: &impl Matcher, - ids_in_formatted: &[FieldId], - formatted_options: &HashMap, + formatted_options: &BTreeMap, ) -> anyhow::Result { let mut document = Document::new(); - if !formatted_options.is_empty() { - for field in ids_in_formatted { - if let Some(value) = obkv.get(*field) { - let mut value: Value = serde_json::from_slice(value)?; + for (id, format) in formatted_options { + if let Some(value) = obkv.get(*id) { + let mut value: Value = serde_json::from_slice(value)?; - if let Some(format) = formatted_options.get(field) { - value = formatter.format_value( - value, - matching_words, - *format, - ); - } + value = formatter.format_value( + value, + matching_words, + *format, + ); - // This unwrap must be safe since we got the ids from the fields_ids_map just - // before. - let key = field_ids_map - .name(*field) - .expect("Missing field name") - .to_string(); + // This unwrap must be safe since we got the ids from the fields_ids_map just + // before. + let key = field_ids_map + .name(*id) + .expect("Missing field name") + .to_string(); - document.insert(key, value); - } + document.insert(key, value); } } @@ -378,7 +393,7 @@ trait Matcher { } #[cfg(test)] -impl Matcher for HashMap<&str, Option> { +impl Matcher for BTreeMap<&str, Option> { fn matches(&self, w: &str) -> Option { self.get(w).cloned().flatten() } @@ -564,8 +579,7 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let ids_in_formatted = Vec::new(); - let formatted_options = HashMap::new(); + let formatted_options = BTreeMap::new(); let matching_words = MatchingWords::default(); @@ -574,7 +588,6 @@ mod test { obkv, &formatter, &matching_words, - &ids_in_formatted, &formatted_options, ) .unwrap(); @@ -582,84 +595,6 @@ mod test { assert!(value.is_empty()); } - #[test] - fn no_formatted_with_ids() { - let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); - - let mut fields = FieldsIdsMap::new(); - let id = fields.insert("test").unwrap(); - - let mut buf = Vec::new(); - let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()) - .unwrap(); - obkv.finish().unwrap(); - - let obkv = obkv::KvReader::new(&buf); - - let ids_in_formatted = vec![id]; - let formatted_options = HashMap::new(); - - let matching_words = MatchingWords::default(); - - let value = format_fields( - &fields, - obkv, - &formatter, - &matching_words, - &ids_in_formatted, - &formatted_options, - ) - .unwrap(); - - assert!(value.is_empty()); - } - - #[test] - fn formatted_with_highlight() { - let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let mut buf = Vec::new(); - let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(title, Value::String("The Hobbit".into()).to_string().as_bytes()) - .unwrap(); - obkv.finish().unwrap(); - obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(author, Value::String("J. R. R. Tolkien".into()).to_string().as_bytes()) - .unwrap(); - obkv.finish().unwrap(); - - let obkv = obkv::KvReader::new(&buf); - - let ids_in_formatted = vec![title, author]; - let mut formatted_options = HashMap::new(); - formatted_options.insert(title, FormatOptions { highlight: true, crop: None }); - - let mut matching_words = HashMap::new(); - matching_words.insert("hobbit", Some(6)); - - let value = format_fields( - &fields, - obkv, - &formatter, - &matching_words, - &ids_in_formatted, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "The Hobbit"); - assert_eq!(value["author"], "J. R. R. Tolkien"); - } - #[test] fn formatted_with_highlight_in_word() { let stop_words = fst::Set::default(); @@ -682,11 +617,11 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let ids_in_formatted = vec![title, author]; - let mut formatted_options = HashMap::new(); + let mut formatted_options = BTreeMap::new(); formatted_options.insert(title, FormatOptions { highlight: true, crop: None }); + formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); - let mut matching_words = HashMap::new(); + let mut matching_words = BTreeMap::new(); matching_words.insert("hobbit", Some(3)); let value = format_fields( @@ -694,7 +629,6 @@ mod test { obkv, &formatter, &matching_words, - &ids_in_formatted, &formatted_options, ) .unwrap(); @@ -725,11 +659,11 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let ids_in_formatted = vec![title, author]; - let mut formatted_options = HashMap::new(); + let mut formatted_options = BTreeMap::new(); formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(2) }); + formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); - let mut matching_words = HashMap::new(); + let mut matching_words = BTreeMap::new(); matching_words.insert("potter", Some(6)); let value = format_fields( @@ -737,7 +671,6 @@ mod test { obkv, &formatter, &matching_words, - &ids_in_formatted, &formatted_options, ) .unwrap(); @@ -768,11 +701,11 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let ids_in_formatted = vec![title, author]; - let mut formatted_options = HashMap::new(); + let mut formatted_options = BTreeMap::new(); formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(10) }); + formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); - let mut matching_words = HashMap::new(); + let mut matching_words = BTreeMap::new(); matching_words.insert("potter", Some(6)); let value = format_fields( @@ -780,7 +713,6 @@ mod test { obkv, &formatter, &matching_words, - &ids_in_formatted, &formatted_options, ) .unwrap(); @@ -811,11 +743,11 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let ids_in_formatted = vec![title, author]; - let mut formatted_options = HashMap::new(); + let mut formatted_options = BTreeMap::new(); formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(0) }); + formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); - let mut matching_words = HashMap::new(); + let mut matching_words = BTreeMap::new(); matching_words.insert("potter", Some(6)); let value = format_fields( @@ -823,7 +755,6 @@ mod test { obkv, &formatter, &matching_words, - &ids_in_formatted, &formatted_options, ) .unwrap(); @@ -854,11 +785,11 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let ids_in_formatted = vec![title, author]; - let mut formatted_options = HashMap::new(); + let mut formatted_options = BTreeMap::new(); formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(1) }); + formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); - let mut matching_words = HashMap::new(); + let mut matching_words = BTreeMap::new(); matching_words.insert("and", Some(3)); let value = format_fields( @@ -866,7 +797,6 @@ mod test { obkv, &formatter, &matching_words, - &ids_in_formatted, &formatted_options, ) .unwrap(); @@ -897,11 +827,11 @@ mod test { let obkv = obkv::KvReader::new(&buf); - let ids_in_formatted = vec![title, author]; - let mut formatted_options = HashMap::new(); + let mut formatted_options = BTreeMap::new(); formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(9) }); + formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); - let mut matching_words = HashMap::new(); + let mut matching_words = BTreeMap::new(); matching_words.insert("blood", Some(3)); let value = format_fields( @@ -909,7 +839,6 @@ mod test { obkv, &formatter, &matching_words, - &ids_in_formatted, &formatted_options, ) .unwrap(); From 9543ab4db6bf0ed36b1aa5666b54fdb30cd21472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 17 Jun 2021 13:50:49 +0200 Subject: [PATCH 427/522] Use mut instead of returning the hashmap --- meilisearch-http/src/index/search.rs | 32 +++++++++++----------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 40ac1b1d9..e5eb66229 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -144,7 +144,7 @@ impl Index { // These attributes are: // - the attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`) // - the attributes asked to be retrieved: these attributes will not be highlighted/cropped - // But these attributes must be present in displayed attributes + // But these attributes must be also present in displayed attributes let formatted_options = compute_formatted_options( &attr_to_highlight, &attr_to_crop, @@ -212,15 +212,15 @@ fn compute_formatted_options( let mut formatted_options = BTreeMap::new(); - formatted_options = add_highlight_to_formatted_options( - formatted_options, + add_highlight_to_formatted_options( + &mut formatted_options, attr_to_highlight, fields_ids_map, displayed_ids, ); - formatted_options = add_crop_to_formatted_options( - formatted_options, + add_crop_to_formatted_options( + &mut formatted_options, attr_to_crop, query_crop_length, fields_ids_map, @@ -229,8 +229,8 @@ fn compute_formatted_options( // Should not return `_formatted` if no valid attributes to highlight/crop if !formatted_options.is_empty() { - formatted_options = add_non_formatted_ids_to_formatted_options( - formatted_options, + add_non_formatted_ids_to_formatted_options( + &mut formatted_options, to_retrieve_ids, ); } @@ -239,11 +239,11 @@ fn compute_formatted_options( } fn add_highlight_to_formatted_options( - mut formatted_options: BTreeMap, + formatted_options: &mut BTreeMap, attr_to_highlight: &HashSet, fields_ids_map: &FieldsIdsMap, displayed_ids: &BTreeSet, -) -> BTreeMap { +) { for attr in attr_to_highlight { let new_format = FormatOptions { highlight: true, @@ -263,17 +263,15 @@ fn add_highlight_to_formatted_options( } } } - - formatted_options } fn add_crop_to_formatted_options( - mut formatted_options: BTreeMap, + formatted_options: &mut BTreeMap, attr_to_crop: &[String], crop_length: usize, fields_ids_map: &FieldsIdsMap, displayed_ids: &BTreeSet, -) -> BTreeMap { +) { for attr in attr_to_crop { let mut attr_name = attr.clone(); let mut attr_len = crop_length; @@ -311,14 +309,12 @@ fn add_crop_to_formatted_options( } } } - - formatted_options } fn add_non_formatted_ids_to_formatted_options( - mut formatted_options: BTreeMap, + formatted_options: &mut BTreeMap, to_retrieve_ids: &BTreeSet -) -> BTreeMap { +) { for id in to_retrieve_ids { formatted_options .entry(*id) @@ -327,8 +323,6 @@ fn add_non_formatted_ids_to_formatted_options( crop: None, }); } - - formatted_options } fn make_document( From 33e55bd82e8ecbe2514aea5542a9579c98609f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 17 Jun 2021 16:59:01 +0200 Subject: [PATCH 428/522] Refactor the crop --- meilisearch-http/src/index/search.rs | 128 ++++++++++++++++++--------- 1 file changed, 87 insertions(+), 41 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index e5eb66229..51dc1dd85 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,5 +1,5 @@ use std::borrow::Cow; -use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::time::Instant; use anyhow::bail; @@ -273,16 +273,13 @@ fn add_crop_to_formatted_options( displayed_ids: &BTreeSet, ) { for attr in attr_to_crop { - let mut attr_name = attr.clone(); - let mut attr_len = crop_length; - - let mut split = attr_name.rsplitn(2, ':'); - attr_name = match split.next().zip(split.next()) { + let mut split = attr.rsplitn(2, ':'); + let (attr_name, attr_len) = match split.next().zip(split.next()) { Some((len, name)) => { - attr_len = len.parse().unwrap_or(crop_length); - name.to_string() + let crop_len = len.parse::().unwrap_or(crop_length); + (name, crop_len) }, - None => attr_name, + None => (attr.as_str(), crop_length), }; if attr_name == "*" { @@ -452,42 +449,49 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { let tokens: Box> = match format_options.crop { Some(crop_len) => { - let mut buffer = VecDeque::new(); + let mut buffer = Vec::new(); let mut tokens = analyzed.reconstruct().peekable(); - let mut taken_before = 0; + while let Some((word, token)) = tokens.next_if(|(_, token)| matcher.matches(token.text()).is_none()) { - buffer.push_back((word, token)); - taken_before += word.chars().count(); - while taken_before > crop_len { - // Around to the previous word - if let Some((word, _)) = buffer.front() { - if taken_before - word.chars().count() < crop_len { - break; - } - } - if let Some((word, _)) = buffer.pop_front() { - taken_before -= word.chars().count(); - } + buffer.push((word, token)); + } + + match tokens.next() { + Some(token) => { + let mut total_len: usize = buffer.iter().map(|(word, _)| word.len()).sum(); + let before_iter = buffer.into_iter().skip_while(move |(word, _)| { + total_len -= word.len(); + let take = total_len >= crop_len; + take + }); + + let mut taken_after = 0; + let after_iter = tokens + .take_while(move |(word, _)| { + let take = taken_after < crop_len; + taken_after += word.chars().count(); + take + }); + + let iter = before_iter + .chain(Some(token)) + .chain(after_iter); + + Box::new(iter) + + }, + // If no word matches in the attribute + None => { + let mut count = 0; + let iter = buffer.into_iter().take_while(move |(word, _)| { + let take = count < crop_len; + count += word.len(); + take + }); + + Box::new(iter) } } - - if let Some(token) = tokens.next() { - buffer.push_back(token); - } - - let mut taken_after = 0; - let after_iter = tokens - .take_while(move |(word, _)| { - let take = taken_after < crop_len; - taken_after += word.chars().count(); - take - }); - - let iter = buffer - .into_iter() - .chain(after_iter); - - Box::new(iter) } None => Box::new(analyzed.reconstruct()), }; @@ -757,6 +761,48 @@ mod test { assert_eq!(value["author"], "J. K. Rowling"); } + #[test] + fn formatted_with_crop_and_no_match() { + let stop_words = fst::Set::default(); + let formatter = + Formatter::new(&stop_words, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(6) }); + formatted_options.insert(author, FormatOptions { highlight: false, crop: Some(20) }); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("rowling", Some(3)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Harry "); + assert_eq!(value["author"], "J. K. Rowling"); + } + #[test] fn formatted_with_crop_and_highlight() { let stop_words = fst::Set::default(); From e4b3d35ed8e45b18f3ea881575ff0a9cac98d3c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 17 Jun 2021 17:03:43 +0200 Subject: [PATCH 429/522] Fix clippy errors --- meilisearch-http/src/index/search.rs | 3 +-- meilisearch-http/src/index_controller/index_actor/message.rs | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 51dc1dd85..00ada864f 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -461,8 +461,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { let mut total_len: usize = buffer.iter().map(|(word, _)| word.len()).sum(); let before_iter = buffer.into_iter().skip_while(move |(word, _)| { total_len -= word.len(); - let take = total_len >= crop_len; - take + total_len >= crop_len }); let mut taken_after = 0; diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 377b2c333..e7304d56c 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -8,6 +8,7 @@ use crate::index_controller::{Failed, IndexStats, Processed, Processing}; use super::{IndexMeta, IndexResult, IndexSettings}; +#[allow(clippy::large_enum_variant)] pub enum IndexMsg { CreateIndex { uuid: Uuid, From c5c7e76805236a36d67754cb76b19cdfade2311e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 17 Jun 2021 18:00:02 +0200 Subject: [PATCH 430/522] Update meilisearch-http/src/index/search.rs Co-authored-by: marin --- meilisearch-http/src/index/search.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 00ada864f..3261e9a37 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -477,7 +477,6 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { .chain(after_iter); Box::new(iter) - }, // If no word matches in the attribute None => { From 623b71e81e5f079bfade7e7d84686075873ff414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 17 Jun 2021 18:02:25 +0200 Subject: [PATCH 431/522] Fix clippy errors --- meilisearch-http/src/routes/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index beddf7ee3..4c5ebbe21 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -17,7 +17,7 @@ pub mod settings; pub mod stats; #[derive(Debug, Clone, Serialize, Deserialize)] -#[allow(clippy::clippy::large_enum_variant)] +#[allow(clippy::large_enum_variant)] #[serde(tag = "name")] pub enum UpdateType { ClearAll, From 9e8888b603c6307992aa294dc37561eeeafa723e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 17 Jun 2021 18:50:18 +0200 Subject: [PATCH 432/522] Fix clippy errors --- meilisearch-http/tests/index/stats.rs | 4 ++-- meilisearch-http/tests/stats/mod.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs index d32c06d2b..b0df427b6 100644 --- a/meilisearch-http/tests/index/stats.rs +++ b/meilisearch-http/tests/index/stats.rs @@ -14,7 +14,7 @@ async fn stats() { assert_eq!(code, 200); assert_eq!(response["numberOfDocuments"], 0); - assert_eq!(response["isIndexing"], false); + assert!(response["isIndexing"] == false); assert!(response["fieldsDistribution"] .as_object() .unwrap() @@ -41,7 +41,7 @@ async fn stats() { assert_eq!(code, 200); assert_eq!(response["numberOfDocuments"], 2); - assert_eq!(response["isIndexing"], false); + assert!(response["isIndexing"] == false); assert_eq!(response["fieldsDistribution"]["id"], 2); assert_eq!(response["fieldsDistribution"]["name"], 1); assert_eq!(response["fieldsDistribution"]["age"], 1); diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index f931d5066..7b9ab10b0 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -37,7 +37,7 @@ async fn stats() { assert!(response.get("lastUpdate").is_some()); assert!(response["indexes"].get("test").is_some()); assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 0); - assert_eq!(response["indexes"]["test"]["isIndexing"], false); + assert!(response["indexes"]["test"]["isIndexing"] == false); let last_update = response["lastUpdate"].as_str().unwrap(); @@ -68,7 +68,7 @@ async fn stats() { assert!(response["databaseSize"].as_u64().unwrap() > 0); assert!(response["lastUpdate"].as_str().unwrap() > last_update); assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 2); - assert_eq!(response["indexes"]["test"]["isIndexing"], false); + assert!(response["indexes"]["test"]["isIndexing"] == false); assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["id"], 2); assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["name"], 1); assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["age"], 1); From 5d8a21b0defdd7e4f7f52bfbebb07ce511daf5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 17 Jun 2021 18:51:07 +0200 Subject: [PATCH 433/522] Fix clippy errors --- meilisearch-http/src/routes/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index beddf7ee3..4c5ebbe21 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -17,7 +17,7 @@ pub mod settings; pub mod stats; #[derive(Debug, Clone, Serialize, Deserialize)] -#[allow(clippy::clippy::large_enum_variant)] +#[allow(clippy::large_enum_variant)] #[serde(tag = "name")] pub enum UpdateType { ClearAll, From 3a2e7d3c3bb056c83911ce26f3e04d21555d7158 Mon Sep 17 00:00:00 2001 From: marin postma Date: Sun, 20 Jun 2021 16:59:31 +0200 Subject: [PATCH 434/522] optimize cropping --- meilisearch-http/src/index/search.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index fbc47cd4e..208804803 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -496,18 +496,19 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { tokens .map(|(word, token)| { - if format_options.highlight && token.is_word() && matcher.matches(token.text()).is_some() { - let mut new_word = String::new(); - new_word.push_str(&self.marks.0); - if let Some(match_len) = matcher.matches(token.text()) { + if let Some(match_len) = matcher.matches(token.text()) { + if format_options.highlight && token.is_word() { + let mut new_word = String::new(); + + new_word.push_str(&self.marks.0); new_word.push_str(&word[..match_len]); new_word.push_str(&self.marks.1); new_word.push_str(&word[match_len..]); + + return Cow::Owned(new_word) } - Cow::Owned(new_word) - } else { - Cow::Borrowed(word) } + Cow::Borrowed(word) }) .collect::() } From 58f9974be42e80ac4245311bfdd6b4695fcffd9b Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 14 Jun 2021 21:26:35 +0200 Subject: [PATCH 435/522] remove anyhow refs & implement missing errors --- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/data/mod.rs | 19 +- meilisearch-http/src/data/search.rs | 7 +- meilisearch-http/src/data/updates.rs | 18 +- meilisearch-http/src/error.rs | 310 ++++-------------- .../src/helpers/authentication.rs | 6 +- meilisearch-http/src/helpers/compression.rs | 4 +- meilisearch-http/src/index/dump.rs | 19 +- meilisearch-http/src/index/error.rs | 60 ++++ meilisearch-http/src/index/mod.rs | 68 ++-- meilisearch-http/src/index/search.rs | 57 ++-- meilisearch-http/src/index/update_handler.rs | 3 +- meilisearch-http/src/index/updates.rs | 46 ++- .../src/index_controller/dump_actor/actor.rs | 11 +- .../src/index_controller/dump_actor/error.rs | 51 +++ .../dump_actor/handle_impl.rs | 9 +- .../index_controller/dump_actor/loaders/v1.rs | 6 +- .../index_controller/dump_actor/loaders/v2.rs | 2 +- .../index_controller/dump_actor/message.rs | 7 +- .../src/index_controller/dump_actor/mod.rs | 38 +-- .../src/index_controller/error.rs | 40 +++ .../src/index_controller/index_actor/actor.rs | 60 ++-- .../src/index_controller/index_actor/error.rs | 49 +++ .../index_actor/handle_impl.rs | 29 +- .../index_controller/index_actor/message.rs | 5 +- .../src/index_controller/index_actor/mod.rs | 94 ++---- .../src/index_controller/index_actor/store.rs | 20 +- meilisearch-http/src/index_controller/mod.rs | 50 +-- .../src/index_controller/snapshot.rs | 15 +- .../index_controller/update_actor/actor.rs | 11 +- .../index_controller/update_actor/error.rs | 64 ++++ .../update_actor/handle_impl.rs | 71 +--- .../index_controller/update_actor/message.rs | 3 +- .../src/index_controller/update_actor/mod.rs | 47 +-- .../update_actor/store/dump.rs | 14 +- .../update_actor/store/mod.rs | 21 +- .../uuid_resolver/handle_impl.rs | 8 +- .../src/index_controller/uuid_resolver/mod.rs | 23 +- .../index_controller/uuid_resolver/store.rs | 4 +- meilisearch-http/src/routes/search.rs | 4 +- 40 files changed, 707 insertions(+), 668 deletions(-) create mode 100644 meilisearch-http/src/index/error.rs create mode 100644 meilisearch-http/src/index_controller/dump_actor/error.rs create mode 100644 meilisearch-http/src/index_controller/error.rs create mode 100644 meilisearch-http/src/index_controller/index_actor/error.rs create mode 100644 meilisearch-http/src/index_controller/update_actor/error.rs diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 70ab30d34..485ccf972 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -27,7 +27,7 @@ actix-http = { version = "=3.0.0-beta.6" } actix-service = "2.0.0" actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] } actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true } -anyhow = "1.0.36" +#anyhow = "1.0.36" async-stream = "0.3.0" async-trait = "0.1.42" arc-swap = "1.2.0" diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 9f8a688bc..634216ade 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -6,6 +6,7 @@ use sha2::Digest; use crate::index::{Checked, Settings}; use crate::index_controller::{ DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats, + error::Result }; use crate::option::Opt; @@ -56,7 +57,7 @@ impl ApiKeys { } impl Data { - pub fn new(options: Opt) -> anyhow::Result { + pub fn new(options: Opt) -> std::result::Result> { let path = options.db_path.clone(); let index_controller = IndexController::new(&path, &options)?; @@ -79,15 +80,15 @@ impl Data { Ok(Data { inner }) } - pub async fn settings(&self, uid: String) -> anyhow::Result> { + pub async fn settings(&self, uid: String) -> Result> { self.index_controller.settings(uid).await } - pub async fn list_indexes(&self) -> anyhow::Result> { + pub async fn list_indexes(&self) -> Result> { self.index_controller.list_indexes().await } - pub async fn index(&self, uid: String) -> anyhow::Result { + pub async fn index(&self, uid: String) -> Result { self.index_controller.get_index(uid).await } @@ -95,7 +96,7 @@ impl Data { &self, uid: String, primary_key: Option, - ) -> anyhow::Result { + ) -> Result { let settings = IndexSettings { uid: Some(uid), primary_key, @@ -105,19 +106,19 @@ impl Data { Ok(meta) } - pub async fn get_index_stats(&self, uid: String) -> anyhow::Result { + pub async fn get_index_stats(&self, uid: String) -> Result { Ok(self.index_controller.get_index_stats(uid).await?) } - pub async fn get_all_stats(&self) -> anyhow::Result { + pub async fn get_all_stats(&self) -> Result { Ok(self.index_controller.get_all_stats().await?) } - pub async fn create_dump(&self) -> anyhow::Result { + pub async fn create_dump(&self) -> Result { Ok(self.index_controller.create_dump().await?) } - pub async fn dump_status(&self, uid: String) -> anyhow::Result { + pub async fn dump_status(&self, uid: String) -> Result { Ok(self.index_controller.dump_info(uid).await?) } diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index 1a998b997..30645cfaf 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -2,13 +2,14 @@ use serde_json::{Map, Value}; use super::Data; use crate::index::{SearchQuery, SearchResult}; +use crate::index_controller::error::Result; impl Data { pub async fn search( &self, index: String, search_query: SearchQuery, - ) -> anyhow::Result { + ) -> Result { self.index_controller.search(index, search_query).await } @@ -18,7 +19,7 @@ impl Data { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> anyhow::Result>> { + ) -> Result>> { self.index_controller .documents(index, offset, limit, attributes_to_retrieve) .await @@ -29,7 +30,7 @@ impl Data { index: String, document_id: String, attributes_to_retrieve: Option>, - ) -> anyhow::Result> { + ) -> Result> { self.index_controller .document(index, document_id, attributes_to_retrieve) .await diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index 23949aa86..e2b9f6629 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -3,7 +3,7 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use super::Data; use crate::index::{Checked, Settings}; -use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus}; +use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus, error::Result}; impl Data { pub async fn add_documents( @@ -13,7 +13,7 @@ impl Data { format: UpdateFormat, stream: Payload, primary_key: Option, - ) -> anyhow::Result { + ) -> Result { let update_status = self .index_controller .add_documents(index, method, format, stream, primary_key) @@ -26,7 +26,7 @@ impl Data { index: String, settings: Settings, create: bool, - ) -> anyhow::Result { + ) -> Result { let update = self .index_controller .update_settings(index, settings, create) @@ -34,7 +34,7 @@ impl Data { Ok(update) } - pub async fn clear_documents(&self, index: String) -> anyhow::Result { + pub async fn clear_documents(&self, index: String) -> Result { let update = self.index_controller.clear_documents(index).await?; Ok(update) } @@ -43,7 +43,7 @@ impl Data { &self, index: String, document_ids: Vec, - ) -> anyhow::Result { + ) -> Result { let update = self .index_controller .delete_documents(index, document_ids) @@ -51,16 +51,16 @@ impl Data { Ok(update) } - pub async fn delete_index(&self, index: String) -> anyhow::Result<()> { + pub async fn delete_index(&self, index: String) -> Result<()> { self.index_controller.delete_index(index).await?; Ok(()) } - pub async fn get_update_status(&self, index: String, uid: u64) -> anyhow::Result { + pub async fn get_update_status(&self, index: String, uid: u64) -> Result { self.index_controller.update_status(index, uid).await } - pub async fn get_updates_status(&self, index: String) -> anyhow::Result> { + pub async fn get_updates_status(&self, index: String) -> Result> { self.index_controller.all_update_status(index).await } @@ -69,7 +69,7 @@ impl Data { uid: String, primary_key: Option, new_uid: Option, - ) -> anyhow::Result { + ) -> Result { let settings = IndexSettings { uid: new_uid, primary_key, diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 07bd96fb9..04e2c52ca 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -1,15 +1,33 @@ use std::error; +use std::error::Error; use std::fmt; use actix_web as aweb; use actix_web::body::Body; use actix_web::dev::BaseHttpResponseBuilder; -use actix_web::error::{JsonPayloadError, QueryPayloadError}; -use actix_web::http::Error as HttpError; use actix_web::http::StatusCode; use meilisearch_error::{Code, ErrorCode}; use serde::ser::{Serialize, SerializeStruct, Serializer}; +use crate::index_controller::error::IndexControllerError; + +#[derive(Debug, thiserror::Error)] +pub enum AuthenticationError { + #[error("You must have an authorization token")] + MissingAuthorizationHeader, + #[error("Invalid API key")] + InvalidToken(String), +} + +impl ErrorCode for AuthenticationError { + fn error_code(&self) -> Code { + match self { + AuthenticationError ::MissingAuthorizationHeader => Code::MissingAuthorizationHeader, + AuthenticationError::InvalidToken(_) => Code::InvalidToken, + } + } +} + #[derive(Debug)] pub struct ResponseError { inner: Box, @@ -29,30 +47,26 @@ impl fmt::Display for ResponseError { } } -// TODO: remove this when implementing actual error handling -impl From for ResponseError { - fn from(other: anyhow::Error) -> ResponseError { - ResponseError { - inner: Box::new(Error::NotFound(other.to_string())), - } - } +macro_rules! response_error { + ($($other:path), *) => { + $( + impl From<$other> for ResponseError { + fn from(error: $other) -> ResponseError { + ResponseError { + inner: Box::new(error), + } + } + } + + )* + }; } -impl From for ResponseError { - fn from(error: Error) -> ResponseError { - ResponseError { - inner: Box::new(error), - } - } -} +response_error!( + IndexControllerError, + AuthenticationError +); -impl From for ResponseError { - fn from(err: FacetCountError) -> ResponseError { - ResponseError { - inner: Box::new(err), - } - } -} impl Serialize for ResponseError { fn serialize(&self, serializer: S) -> Result @@ -83,239 +97,35 @@ impl aweb::error::ResponseError for ResponseError { } #[derive(Debug)] -pub enum Error { - BadParameter(String, String), - BadRequest(String), - CreateIndex(String), - DocumentNotFound(String), - IndexNotFound(String), - IndexAlreadyExists(String), - Internal(String), - InvalidIndexUid, - InvalidToken(String), - MissingAuthorizationHeader, - NotFound(String), - OpenIndex(String), - RetrieveDocument(u32, String), - SearchDocuments(String), - PayloadTooLarge, - UnsupportedMediaType, - DumpAlreadyInProgress, - DumpProcessFailed(String), -} +struct PayloadError(E); -impl error::Error for Error {} - -impl ErrorCode for Error { - fn error_code(&self) -> Code { - use Error::*; - match self { - BadParameter(_, _) => Code::BadParameter, - BadRequest(_) => Code::BadRequest, - CreateIndex(_) => Code::CreateIndex, - DocumentNotFound(_) => Code::DocumentNotFound, - IndexNotFound(_) => Code::IndexNotFound, - IndexAlreadyExists(_) => Code::IndexAlreadyExists, - Internal(_) => Code::Internal, - InvalidIndexUid => Code::InvalidIndexUid, - InvalidToken(_) => Code::InvalidToken, - MissingAuthorizationHeader => Code::MissingAuthorizationHeader, - NotFound(_) => Code::NotFound, - OpenIndex(_) => Code::OpenIndex, - RetrieveDocument(_, _) => Code::RetrieveDocument, - SearchDocuments(_) => Code::SearchDocuments, - PayloadTooLarge => Code::PayloadTooLarge, - UnsupportedMediaType => Code::UnsupportedMediaType, - _ => unreachable!() - //DumpAlreadyInProgress => Code::DumpAlreadyInProgress, - //DumpProcessFailed(_) => Code::DumpProcessFailed, - } - } -} - -#[derive(Debug)] -pub enum FacetCountError { - AttributeNotSet(String), - SyntaxError(String), - UnexpectedToken { - found: String, - expected: &'static [&'static str], - }, - NoFacetSet, -} - -impl error::Error for FacetCountError {} - -impl ErrorCode for FacetCountError { - fn error_code(&self) -> Code { - Code::BadRequest - } -} - -impl FacetCountError { - pub fn unexpected_token( - found: impl ToString, - expected: &'static [&'static str], - ) -> FacetCountError { - let found = found.to_string(); - FacetCountError::UnexpectedToken { expected, found } - } -} - -impl From for FacetCountError { - fn from(other: serde_json::error::Error) -> FacetCountError { - FacetCountError::SyntaxError(other.to_string()) - } -} - -impl fmt::Display for FacetCountError { +impl fmt::Display for PayloadError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use FacetCountError::*; + std::fmt::Display::fmt(&self.0, f) + } +} - match self { - AttributeNotSet(attr) => write!(f, "Attribute {} is not set as facet", attr), - SyntaxError(msg) => write!(f, "Syntax error: {}", msg), - UnexpectedToken { expected, found } => { - write!(f, "Unexpected {} found, expected {:?}", found, expected) - } - NoFacetSet => write!(f, "Can't perform facet count, as no facet is set"), +impl Error for PayloadError {} + +impl ErrorCode for PayloadError { + fn error_code(&self) -> Code { + Code::Internal + } +} + +impl From> for ResponseError +where E: Error + Sync + Send + 'static +{ + fn from(other: PayloadError) -> Self { + ResponseError { + inner: Box::new(other), } } } -impl Error { - pub fn internal(err: impl fmt::Display) -> Error { - Error::Internal(err.to_string()) - } - - pub fn bad_request(err: impl fmt::Display) -> Error { - Error::BadRequest(err.to_string()) - } - - pub fn missing_authorization_header() -> Error { - Error::MissingAuthorizationHeader - } - - pub fn invalid_token(err: impl fmt::Display) -> Error { - Error::InvalidToken(err.to_string()) - } - - pub fn not_found(err: impl fmt::Display) -> Error { - Error::NotFound(err.to_string()) - } - - pub fn index_not_found(err: impl fmt::Display) -> Error { - Error::IndexNotFound(err.to_string()) - } - - pub fn document_not_found(err: impl fmt::Display) -> Error { - Error::DocumentNotFound(err.to_string()) - } - - pub fn bad_parameter(param: impl fmt::Display, err: impl fmt::Display) -> Error { - Error::BadParameter(param.to_string(), err.to_string()) - } - - pub fn open_index(err: impl fmt::Display) -> Error { - Error::OpenIndex(err.to_string()) - } - - pub fn create_index(err: impl fmt::Display) -> Error { - Error::CreateIndex(err.to_string()) - } - - pub fn invalid_index_uid() -> Error { - Error::InvalidIndexUid - } - - pub fn retrieve_document(doc_id: u32, err: impl fmt::Display) -> Error { - Error::RetrieveDocument(doc_id, err.to_string()) - } - - pub fn search_documents(err: impl fmt::Display) -> Error { - Error::SearchDocuments(err.to_string()) - } - - pub fn dump_conflict() -> Error { - Error::DumpAlreadyInProgress - } - - pub fn dump_failed(message: String) -> Error { - Error::DumpProcessFailed(message) - } -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::BadParameter(param, err) => write!(f, "Url parameter {} error: {}", param, err), - Self::BadRequest(err) => f.write_str(err), - Self::CreateIndex(err) => write!(f, "Impossible to create index; {}", err), - Self::DocumentNotFound(document_id) => write!(f, "Document with id {} not found", document_id), - Self::IndexNotFound(index_uid) => write!(f, "Index {} not found", index_uid), - Self::IndexAlreadyExists(index_uid) => write!(f, "Index {} already exists", index_uid), - Self::Internal(err) => f.write_str(err), - Self::InvalidIndexUid => f.write_str("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."), - Self::InvalidToken(err) => write!(f, "Invalid API key: {}", err), - Self::MissingAuthorizationHeader => f.write_str("You must have an authorization token"), - Self::NotFound(err) => write!(f, "{} not found", err), - Self::OpenIndex(err) => write!(f, "Impossible to open index; {}", err), - Self::RetrieveDocument(id, err) => write!(f, "Impossible to retrieve the document with id: {}; {}", id, err), - Self::SearchDocuments(err) => write!(f, "Impossible to search documents; {}", err), - Self::PayloadTooLarge => f.write_str("Payload too large"), - Self::UnsupportedMediaType => f.write_str("Unsupported media type"), - Self::DumpAlreadyInProgress => f.write_str("Another dump is already in progress"), - Self::DumpProcessFailed(message) => write!(f, "Dump process failed: {}", message), - } - } -} - -impl From for Error { - fn from(err: std::io::Error) -> Error { - Error::Internal(err.to_string()) - } -} - -impl From for Error { - fn from(err: HttpError) -> Error { - Error::Internal(err.to_string()) - } -} - -impl From for Error { - fn from(err: serde_json::error::Error) -> Error { - Error::Internal(err.to_string()) - } -} - -impl From for Error { - fn from(err: JsonPayloadError) -> Error { - match err { - JsonPayloadError::Deserialize(err) => { - Error::BadRequest(format!("Invalid JSON: {}", err)) - } - JsonPayloadError::Overflow => Error::PayloadTooLarge, - JsonPayloadError::ContentType => Error::UnsupportedMediaType, - JsonPayloadError::Payload(err) => { - Error::BadRequest(format!("Problem while decoding the request: {}", err)) - } - e => Error::Internal(format!("Unexpected Json error: {}", e)), - } - } -} - -impl From for Error { - fn from(err: QueryPayloadError) -> Error { - match err { - QueryPayloadError::Deserialize(err) => { - Error::BadRequest(format!("Invalid query parameters: {}", err)) - } - e => Error::Internal(format!("Unexpected query payload error: {}", e)), - } - } -} - -pub fn payload_error_handler>(err: E) -> ResponseError { - let error: Error = err.into(); +pub fn payload_error_handler(err: E) -> ResponseError +where E: Error + Sync + Send + 'static +{ + let error = PayloadError(err); error.into() } diff --git a/meilisearch-http/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs index 54d5488f4..5577b9c1d 100644 --- a/meilisearch-http/src/helpers/authentication.rs +++ b/meilisearch-http/src/helpers/authentication.rs @@ -9,7 +9,7 @@ use futures::future::{ok, Future, Ready}; use futures::ready; use pin_project::pin_project; -use crate::error::{Error, ResponseError}; +use crate::error::{ResponseError, AuthenticationError}; use crate::Data; #[derive(Clone, Copy)] @@ -117,7 +117,7 @@ where AuthProj::NoHeader(req) => { match req.take() { Some(req) => { - let response = ResponseError::from(Error::MissingAuthorizationHeader); + let response = ResponseError::from(AuthenticationError::MissingAuthorizationHeader); let response = response.error_response(); let response = req.into_response(response); Poll::Ready(Ok(response)) @@ -134,7 +134,7 @@ where .get("X-Meili-API-Key") .map(|h| h.to_str().map(String::from).unwrap_or_default()) .unwrap_or_default(); - let response = ResponseError::from(Error::InvalidToken(bad_token)); + let response = ResponseError::from(AuthenticationError::InvalidToken(bad_token)); let response = response.error_response(); let response = req.into_response(response); Poll::Ready(Ok(response)) diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index c4747cb21..55861ca53 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -5,7 +5,7 @@ use std::path::Path; use flate2::{read::GzDecoder, write::GzEncoder, Compression}; use tar::{Archive, Builder}; -pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { +pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Box> { let mut f = File::create(dest)?; let gz_encoder = GzEncoder::new(&mut f, Compression::default()); let mut tar_encoder = Builder::new(gz_encoder); @@ -16,7 +16,7 @@ pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Resul Ok(()) } -pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { +pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Box> { let f = File::open(&src)?; let gz = GzDecoder::new(f); let mut ar = Archive::new(gz); diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 13e6cbc02..05202be2d 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -3,7 +3,6 @@ use std::io::{BufRead, BufReader, Write}; use std::path::Path; use std::sync::Arc; -use anyhow::{bail, Context}; use heed::RoTxn; use indexmap::IndexMap; use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; @@ -12,6 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::option::IndexerOpts; use super::{update_handler::UpdateHandler, Index, Settings, Unchecked}; +use super::error::{IndexError, Result}; #[derive(Serialize, Deserialize)] struct DumpMeta { @@ -23,7 +23,7 @@ const META_FILE_NAME: &str = "meta.json"; const DATA_FILE_NAME: &str = "documents.jsonl"; impl Index { - pub fn dump(&self, path: impl AsRef) -> anyhow::Result<()> { + pub fn dump(&self, path: impl AsRef) -> Result<()> { // acquire write txn make sure any ongoing write is finished before we start. let txn = self.env.write_txn()?; @@ -33,11 +33,12 @@ impl Index { Ok(()) } - fn dump_documents(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { + fn dump_documents(&self, txn: &RoTxn, path: impl AsRef) -> Result<()> { let document_file_path = path.as_ref().join(DATA_FILE_NAME); let mut document_file = File::create(&document_file_path)?; - let documents = self.all_documents(txn)?; + let documents = self.all_documents(txn) + .map_err(|e| IndexError::Internal(e.into()))?; let fields_ids_map = self.fields_ids_map(txn)?; // dump documents @@ -60,7 +61,7 @@ impl Index { Ok(()) } - fn dump_meta(&self, txn: &RoTxn, path: impl AsRef) -> anyhow::Result<()> { + fn dump_meta(&self, txn: &RoTxn, path: impl AsRef) -> Result<()> { let meta_file_path = path.as_ref().join(META_FILE_NAME); let mut meta_file = File::create(&meta_file_path)?; @@ -81,11 +82,13 @@ impl Index { dst: impl AsRef, size: usize, indexing_options: &IndexerOpts, - ) -> anyhow::Result<()> { + ) -> std::result::Result<(), Box> { let dir_name = src .as_ref() .file_name() - .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?; + // TODO: remove + //.with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?; + .unwrap(); let dst_dir_path = dst.as_ref().join("indexes").join(dir_name); create_dir_all(&dst_dir_path)?; @@ -124,7 +127,7 @@ impl Index { match Arc::try_unwrap(index.0) { Ok(inner) => inner.prepare_for_closing().wait(), - Err(_) => bail!("Could not close index properly."), + Err(_) => todo!("Could not close index properly."), } Ok(()) diff --git a/meilisearch-http/src/index/error.rs b/meilisearch-http/src/index/error.rs new file mode 100644 index 000000000..cfda813a1 --- /dev/null +++ b/meilisearch-http/src/index/error.rs @@ -0,0 +1,60 @@ +use std::error::Error; + +use meilisearch_error::{Code, ErrorCode}; +use serde_json::Value; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum IndexError { + #[error("Internal error: {0}")] + Internal(Box), + #[error("Document with id {0} not found.")] + DocumentNotFound(String), + #[error("error with facet: {0}")] + Facet(#[from] FacetError), +} + +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for IndexError { + fn from(other: $other) -> Self { + Self::Internal(Box::new(other)) + } + } + )* + } +} + +internal_error!( + std::io::Error, + heed::Error, + fst::Error, + serde_json::Error +); + +impl ErrorCode for IndexError { + fn error_code(&self) -> Code { + match self { + IndexError::Internal(_) => Code::Internal, + IndexError::DocumentNotFound(_) => Code::DocumentNotFound, + IndexError::Facet(e) => e.error_code(), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum FacetError { + #[error("Invalid facet expression, expected {}, found: {1}", .0.join(", "))] + InvalidExpression(&'static [&'static str], Value) +} + +impl ErrorCode for FacetError { + fn error_code(&self) -> Code { + match self { + FacetError::InvalidExpression(_, _) => Code::Facet, + } + } +} + diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 56958760a..816629524 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -5,19 +5,24 @@ use std::ops::Deref; use std::path::Path; use std::sync::Arc; -use anyhow::{bail, Context}; use heed::{EnvOpenOptions, RoTxn}; use milli::obkv_to_json; +use serde::{de::Deserializer, Deserialize}; use serde_json::{Map, Value}; use crate::helpers::EnvSizer; +use error::Result; + pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; -use serde::{de::Deserializer, Deserialize}; pub use updates::{Checked, Facets, Settings, Unchecked}; +use self::error::IndexError; + +pub mod error; +pub mod update_handler; + mod dump; mod search; -pub mod update_handler; mod updates; pub type Document = Map; @@ -33,7 +38,7 @@ impl Deref for Index { } } -pub fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> +pub fn deserialize_some<'de, T, D>(deserializer: D) -> std::result::Result, D::Error> where T: Deserialize<'de>, D: Deserializer<'de>, @@ -42,20 +47,21 @@ where } impl Index { - pub fn open(path: impl AsRef, size: usize) -> anyhow::Result { + pub fn open(path: impl AsRef, size: usize) -> Result { create_dir_all(&path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options, &path)?; + let index = + milli::Index::new(options, &path).map_err(|e| IndexError::Internal(e.into()))?; Ok(Index(Arc::new(index))) } - pub fn settings(&self) -> anyhow::Result> { + pub fn settings(&self) -> Result> { let txn = self.read_txn()?; self.settings_txn(&txn) } - pub fn settings_txn(&self, txn: &RoTxn) -> anyhow::Result> { + pub fn settings_txn(&self, txn: &RoTxn) -> Result> { let displayed_attributes = self .displayed_fields(&txn)? .map(|fields| fields.into_iter().map(String::from).collect()); @@ -65,7 +71,8 @@ impl Index { .map(|fields| fields.into_iter().map(String::from).collect()); let faceted_attributes = self - .faceted_fields(&txn)? + .faceted_fields(&txn) + .map_err(|e| IndexError::Internal(Box::new(e)))? .into_iter() .collect(); @@ -76,8 +83,9 @@ impl Index { .collect(); let stop_words = self - .stop_words(&txn)? - .map(|stop_words| -> anyhow::Result> { + .stop_words(&txn) + .map_err(|e| IndexError::Internal(e.into()))? + .map(|stop_words| -> Result> { Ok(stop_words.stream().into_strs()?.into_iter().collect()) }) .transpose()? @@ -114,12 +122,13 @@ impl Index { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> anyhow::Result>> { + ) -> Result>> { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = - self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = self + .fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map) + .map_err(|e| IndexError::Internal(e.into()))?; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -127,7 +136,8 @@ impl Index { for entry in iter { let (_id, obkv) = entry?; - let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?; + let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv) + .map_err(|e| IndexError::Internal(e.into()))?; documents.push(object); } @@ -138,28 +148,35 @@ impl Index { &self, doc_id: String, attributes_to_retrieve: Option>, - ) -> anyhow::Result> { + ) -> Result> { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = - self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = self + .fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map) + .map_err(|e| IndexError::Internal(e.into()))?; let internal_id = self - .external_documents_ids(&txn)? + .external_documents_ids(&txn) + .map_err(|e| IndexError::Internal(e.into()))? .get(doc_id.as_bytes()) - .with_context(|| format!("Document with id {} not found", doc_id))?; + .ok_or_else(|| IndexError::DocumentNotFound(doc_id.clone()))?; let document = self - .documents(&txn, std::iter::once(internal_id))? + .documents(&txn, std::iter::once(internal_id)) + .map_err(|e| IndexError::Internal(e.into()))? .into_iter() .next() .map(|(_, d)| d); match document { - Some(document) => Ok(obkv_to_json(&fields_to_display, &fields_ids_map, document)?), - None => bail!("Document with id {} not found", doc_id), + Some(document) => { + let document = obkv_to_json(&fields_to_display, &fields_ids_map, document) + .map_err(|e| IndexError::Internal(e.into()))?; + Ok(document) + } + None => Err(IndexError::DocumentNotFound(doc_id)), } } @@ -172,8 +189,9 @@ impl Index { txn: &heed::RoTxn, attributes_to_retrieve: &Option>, fields_ids_map: &milli::FieldsIdsMap, - ) -> anyhow::Result> { - let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { + ) -> Result> { + let mut displayed_fields_ids = match self.displayed_fields_ids(&txn) + .map_err(|e| IndexError::Internal(Box::new(e)))? { Some(ids) => ids.into_iter().collect::>(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index fbc47cd4e..3122f784e 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -2,7 +2,6 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::time::Instant; -use anyhow::bail; use either::Either; use heed::RoTxn; use indexmap::IndexMap; @@ -11,6 +10,9 @@ use milli::{FilterCondition, FieldId, FieldsIdsMap, MatchingWords}; use serde::{Deserialize, Serialize}; use serde_json::Value; +use crate::index::error::FacetError; + +use super::error::{IndexError, Result}; use super::Index; pub type Document = IndexMap; @@ -71,7 +73,7 @@ struct FormatOptions { } impl Index { - pub fn perform_search(&self, query: SearchQuery) -> anyhow::Result { + pub fn perform_search(&self, query: SearchQuery) -> Result { let before_search = Instant::now(); let rtxn = self.read_txn()?; @@ -95,12 +97,14 @@ impl Index { matching_words, candidates, .. - } = search.execute()?; - let mut documents = Vec::new(); + } = search + .execute() + .map_err(|e| IndexError::Internal(e.into()))?; let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); let displayed_ids = self .displayed_fields_ids(&rtxn)? + .map_err(|e| IndexError::Internal(Box::new(e)))? .map(|fields| fields.into_iter().collect::>()) .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect()); @@ -158,6 +162,8 @@ impl Index { let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut documents = Vec::new(); + for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?; let formatted = format_fields( @@ -167,6 +173,7 @@ impl Index { &matching_words, &formatted_options, )?; + let hit = SearchHit { document, formatted, @@ -182,7 +189,12 @@ impl Index { if fields.iter().all(|f| f != "*") { facet_distribution.facets(fields); } - Some(facet_distribution.candidates(candidates).execute()?) + let distribution = facet_distribution + .candidates(candidates) + .execute() + .map_err(|e| IndexError::Internal(e.into()))?; + + Some(distribution) } None => None, }; @@ -326,7 +338,7 @@ fn make_document( attributes_to_retrieve: &BTreeSet, field_ids_map: &FieldsIdsMap, obkv: obkv::KvReader, -) -> anyhow::Result { +) -> Result { let mut document = Document::new(); for attr in attributes_to_retrieve { if let Some(value) = obkv.get(*attr) { @@ -351,7 +363,7 @@ fn format_fields>( formatter: &Formatter, matching_words: &impl Matcher, formatted_options: &BTreeMap, -) -> anyhow::Result { +) -> Result { let mut document = Document::new(); for (id, format) in formatted_options { @@ -513,15 +525,15 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { } } -fn parse_filter( - facets: &Value, - index: &Index, - txn: &RoTxn, -) -> anyhow::Result> { +fn parse_filter(facets: &Value, index: &Index, txn: &RoTxn) -> Result> { match facets { - Value::String(expr) => Ok(Some(FilterCondition::from_str(txn, index, expr)?)), + Value::String(expr) => { + let condition = FilterCondition::from_str(txn, index, expr) + .map_err(|e| IndexError::Internal(e.into()))?; + Ok(Some(condition)) + } Value::Array(arr) => parse_filter_array(txn, index, arr), - v => bail!("Invalid facet expression, expected Array, found: {:?}", v), + v => return Err(FacetError::InvalidExpression(&["Array"], v.clone()).into()), } } @@ -529,7 +541,7 @@ fn parse_filter_array( txn: &RoTxn, index: &Index, arr: &[Value], -) -> anyhow::Result> { +) -> Result> { let mut ands = Vec::new(); for value in arr { match value { @@ -539,19 +551,22 @@ fn parse_filter_array( for value in arr { match value { Value::String(s) => ors.push(s.clone()), - v => bail!("Invalid facet expression, expected String, found: {:?}", v), + v => { + return Err(FacetError::InvalidExpression(&["String"], v.clone()).into()) + } } } ands.push(Either::Left(ors)); } - v => bail!( - "Invalid facet expression, expected String or [String], found: {:?}", - v - ), + v => { + return Err( + FacetError::InvalidExpression(&["String", "[String]"], v.clone()).into(), + ) + } } } - Ok(FilterCondition::from_array(txn, &index.0, ands)?) + FilterCondition::from_array(txn, &index.0, ands).map_err(|e| IndexError::Internal(Box::new(e))) } #[cfg(test)] diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs index 63a074abb..ce6a7e48d 100644 --- a/meilisearch-http/src/index/update_handler.rs +++ b/meilisearch-http/src/index/update_handler.rs @@ -1,7 +1,6 @@ use std::fs::File; use crate::index::Index; -use anyhow::Result; use grenad::CompressionType; use milli::update::UpdateBuilder; use rayon::ThreadPool; @@ -22,7 +21,7 @@ pub struct UpdateHandler { } impl UpdateHandler { - pub fn new(opt: &IndexerOpts) -> anyhow::Result { + pub fn new(opt: &IndexerOpts) -> std::result::Result> { let thread_pool = rayon::ThreadPoolBuilder::new() .num_threads(opt.indexing_jobs.unwrap_or(0)) .build()?; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index ce327520e..04e90f4f4 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -8,11 +8,16 @@ use log::info; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize, Serializer}; +use crate::index::error::IndexError; use crate::index_controller::UpdateResult; +use super::error::Result; use super::{deserialize_some, Index}; -fn serialize_with_wildcard(field: &Option>>, s: S) -> Result +fn serialize_with_wildcard( + field: &Option>>, + s: S, +) -> std::result::Result where S: Serializer, { @@ -174,7 +179,7 @@ impl Index { content: Option, update_builder: UpdateBuilder, primary_key: Option<&str>, - ) -> anyhow::Result { + ) -> Result { let mut txn = self.write_txn()?; let result = self.update_documents_txn( &mut txn, @@ -196,7 +201,7 @@ impl Index { content: Option, update_builder: UpdateBuilder, primary_key: Option<&str>, - ) -> anyhow::Result { + ) -> Result { info!("performing document addition"); // Set the primary key if not set already, ignore if already set. @@ -204,7 +209,8 @@ impl Index { let mut builder = UpdateBuilder::new(0) .settings(txn, &self); builder.set_primary_key(primary_key.to_string()); - builder.execute(|_, _| ())?; + builder.execute(|_, _| ()) + .map_err(|e| IndexError::Internal(Box::new(e)))?; } let mut builder = update_builder.index_documents(txn, self); @@ -216,11 +222,15 @@ impl Index { let gzipped = false; let addition = match content { - Some(content) if gzipped => { - builder.execute(GzDecoder::new(content), indexing_callback)? - } - Some(content) => builder.execute(content, indexing_callback)?, - None => builder.execute(std::io::empty(), indexing_callback)?, + Some(content) if gzipped => builder + .execute(GzDecoder::new(content), indexing_callback) + .map_err(|e| IndexError::Internal(e.into()))?, + Some(content) => builder + .execute(content, indexing_callback) + .map_err(|e| IndexError::Internal(e.into()))?, + None => builder + .execute(std::io::empty(), indexing_callback) + .map_err(|e| IndexError::Internal(e.into()))?, }; info!("document addition done: {:?}", addition); @@ -228,7 +238,7 @@ impl Index { Ok(UpdateResult::DocumentsAddition(addition)) } - pub fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { + pub fn clear_documents(&self, update_builder: UpdateBuilder) -> Result { // We must use the write transaction of the update here. let mut wtxn = self.write_txn()?; let builder = update_builder.clear_documents(&mut wtxn, self); @@ -238,7 +248,7 @@ impl Index { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()), + Err(e) => Err(IndexError::Internal(Box::new(e))), } } @@ -247,7 +257,7 @@ impl Index { txn: &mut heed::RwTxn<'a, 'b>, settings: &Settings, update_builder: UpdateBuilder, - ) -> anyhow::Result { + ) -> Result { // We must use the write transaction of the update here. let mut builder = update_builder.settings(txn, self); @@ -300,7 +310,8 @@ impl Index { builder.execute(|indexing_step, update_id| { info!("update {}: {:?}", update_id, indexing_step) - })?; + }) + .map_err(|e| IndexError::Internal(e.into()))?; Ok(UpdateResult::Other) } @@ -309,7 +320,7 @@ impl Index { &self, settings: &Settings, update_builder: UpdateBuilder, - ) -> anyhow::Result { + ) -> Result { let mut txn = self.write_txn()?; let result = self.update_settings_txn(&mut txn, settings, update_builder)?; txn.commit()?; @@ -320,9 +331,10 @@ impl Index { &self, document_ids: &[String], update_builder: UpdateBuilder, - ) -> anyhow::Result { + ) -> Result { let mut txn = self.write_txn()?; - let mut builder = update_builder.delete_documents(&mut txn, self)?; + let mut builder = update_builder.delete_documents(&mut txn, self) + .map_err(|e| IndexError::Internal(e.into()))?; // We ignore unexisting document ids document_ids.iter().for_each(|id| { @@ -334,7 +346,7 @@ impl Index { .commit() .and(Ok(UpdateResult::DocumentDeletion { deleted })) .map_err(Into::into), - Err(e) => Err(e.into()), + Err(e) => Err(IndexError::Internal(Box::new(e))), } } } diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index c78079de6..a534a3a69 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -10,8 +10,9 @@ use tokio::sync::{mpsc, oneshot, RwLock}; use update_actor::UpdateActorHandle; use uuid_resolver::UuidResolverHandle; -use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus, DumpTask}; +use super::{DumpInfo, DumpMsg, DumpStatus, DumpTask}; use crate::index_controller::{update_actor, uuid_resolver}; +use super::error::{DumpActorError, Result}; pub const CONCURRENT_DUMP_MSG: usize = 10; @@ -95,14 +96,14 @@ where } } - async fn handle_create_dump(&self, ret: oneshot::Sender>) { + async fn handle_create_dump(&self, ret: oneshot::Sender>) { let uid = generate_uid(); let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress); let _lock = match self.lock.try_lock() { Some(lock) => lock, None => { - ret.send(Err(DumpError::DumpAlreadyRunning)) + ret.send(Err(DumpActorError::DumpAlreadyRunning)) .expect("Dump actor is dead"); return; } @@ -147,10 +148,10 @@ where }; } - async fn handle_dump_info(&self, uid: String) -> DumpResult { + async fn handle_dump_info(&self, uid: String) -> Result { match self.dump_infos.read().await.get(&uid) { Some(info) => Ok(info.clone()), - _ => Err(DumpError::DumpDoesNotExist(uid)), + _ => Err(DumpActorError::DumpDoesNotExist(uid)), } } } diff --git a/meilisearch-http/src/index_controller/dump_actor/error.rs b/meilisearch-http/src/index_controller/dump_actor/error.rs new file mode 100644 index 000000000..c0f373181 --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/error.rs @@ -0,0 +1,51 @@ +use meilisearch_error::{Code, ErrorCode}; + +use crate::index_controller::{update_actor::error::UpdateActorError, uuid_resolver::UuidResolverError}; + +pub type Result = std::result::Result; + +#[derive(thiserror::Error, Debug)] +pub enum DumpActorError { + #[error("dump already running")] + DumpAlreadyRunning, + #[error("dump `{0}` does not exist")] + DumpDoesNotExist(String), + #[error("Internal error: {0}")] + Internal(Box), + #[error("error while dumping uuids: {0}")] + UuidResolver(#[from] UuidResolverError), + #[error("error while dumping updates: {0}")] + UpdateActor(#[from] UpdateActorError), +} + +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for DumpActorError { + fn from(other: $other) -> Self { + Self::Internal(Box::new(other)) + } + } + )* + } +} + +internal_error!( + heed::Error, + std::io::Error, + tokio::task::JoinError, + serde_json::error::Error, + tempfile::PersistError +); + +impl ErrorCode for DumpActorError { + fn error_code(&self) -> Code { + match self { + DumpActorError::DumpAlreadyRunning => Code::DumpAlreadyInProgress, + DumpActorError::DumpDoesNotExist(_) => Code::DocumentNotFound, + DumpActorError::Internal(_) => Code::Internal, + DumpActorError::UuidResolver(e) => e.error_code(), + DumpActorError::UpdateActor(e) => e.error_code(), + } + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs index ab91aeae6..b233a4e61 100644 --- a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs @@ -3,7 +3,8 @@ use std::path::Path; use actix_web::web::Bytes; use tokio::sync::{mpsc, oneshot}; -use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg, DumpResult}; +use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg}; +use super::error::Result; #[derive(Clone)] pub struct DumpActorHandleImpl { @@ -12,14 +13,14 @@ pub struct DumpActorHandleImpl { #[async_trait::async_trait] impl DumpActorHandle for DumpActorHandleImpl { - async fn create_dump(&self) -> DumpResult { + async fn create_dump(&self) -> Result { let (ret, receiver) = oneshot::channel(); let msg = DumpMsg::CreateDump { ret }; let _ = self.sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } - async fn dump_info(&self, uid: String) -> DumpResult { + async fn dump_info(&self, uid: String) -> Result { let (ret, receiver) = oneshot::channel(); let msg = DumpMsg::DumpInfo { ret, uid }; let _ = self.sender.send(msg).await; @@ -34,7 +35,7 @@ impl DumpActorHandleImpl { update: crate::index_controller::update_actor::UpdateActorHandleImpl, index_db_size: usize, update_db_size: usize, - ) -> anyhow::Result { + ) -> std::result::Result> { let (sender, receiver) = mpsc::channel(10); let actor = DumpActor::new( receiver, diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index a7f1aa8d1..502a6333e 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -31,7 +31,7 @@ impl MetadataV1 { dst: impl AsRef, size: usize, indexer_options: &IndexerOpts, - ) -> anyhow::Result<()> { + ) -> std::result::Result<(), Box> { info!( "Loading dump, dump database version: {}, dump version: V1", self.db_version @@ -83,7 +83,7 @@ fn load_index( primary_key: Option<&str>, size: usize, indexer_options: &IndexerOpts, -) -> anyhow::Result<()> { +) -> std::result::Result<(), Box> { let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid)); create_dir_all(&index_path)?; @@ -172,7 +172,7 @@ impl From for index_controller::Settings { } /// Extract Settings from `settings.json` file present at provided `dir_path` -fn import_settings(dir_path: impl AsRef) -> anyhow::Result { +fn import_settings(dir_path: impl AsRef) -> std::result::Result> { let path = dir_path.as_ref().join("settings.json"); let file = File::open(path)?; let reader = std::io::BufReader::new(file); diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index eddd8a3b7..7f5420d23 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -34,7 +34,7 @@ impl MetadataV2 { index_db_size: usize, update_db_size: usize, indexing_options: &IndexerOpts, - ) -> anyhow::Result<()> { + ) -> std::result::Result<(), Box> { info!( "Loading dump from {}, dump database version: {}, dump version: V2", self.dump_date, self.db_version diff --git a/meilisearch-http/src/index_controller/dump_actor/message.rs b/meilisearch-http/src/index_controller/dump_actor/message.rs index dff9f5954..b34314eff 100644 --- a/meilisearch-http/src/index_controller/dump_actor/message.rs +++ b/meilisearch-http/src/index_controller/dump_actor/message.rs @@ -1,13 +1,14 @@ use tokio::sync::oneshot; -use super::{DumpInfo, DumpResult}; +use super::DumpInfo; +use super::error::Result; pub enum DumpMsg { CreateDump { - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, DumpInfo { uid: String, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, } diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 66f081e87..35daa0da8 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,13 +1,11 @@ use std::fs::File; use std::path::{Path, PathBuf}; -use anyhow::Context; use chrono::{DateTime, Utc}; -use log::{error, info, warn}; +use log::{info, warn}; #[cfg(test)] use mockall::automock; use serde::{Deserialize, Serialize}; -use thiserror::Error; use tokio::fs::create_dir_all; use loaders::v1::MetadataV1; @@ -18,39 +16,28 @@ pub use handle_impl::*; pub use message::DumpMsg; use super::{update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle}; +use crate::index_controller::dump_actor::error::DumpActorError; use crate::{helpers::compression, option::IndexerOpts}; +use error::Result; mod actor; mod handle_impl; mod loaders; mod message; +pub mod error; const META_FILE_NAME: &str = "metadata.json"; -pub type DumpResult = std::result::Result; - -#[derive(Error, Debug)] -pub enum DumpError { - #[error("error with index: {0}")] - Error(#[from] anyhow::Error), - #[error("Heed error: {0}")] - HeedError(#[from] heed::Error), - #[error("dump already running")] - DumpAlreadyRunning, - #[error("dump `{0}` does not exist")] - DumpDoesNotExist(String), -} - #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait DumpActorHandle { /// Start the creation of a dump /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump] - async fn create_dump(&self) -> DumpResult; + async fn create_dump(&self) -> Result; /// Return the status of an already created dump /// Implementation: [handle_impl::DumpActorHandleImpl::dump_status] - async fn dump_info(&self, uid: String) -> DumpResult; + async fn dump_info(&self, uid: String) -> Result; } #[derive(Debug, Serialize, Deserialize)] @@ -120,7 +107,7 @@ pub fn load_dump( index_db_size: usize, update_db_size: usize, indexer_opts: &IndexerOpts, -) -> anyhow::Result<()> { +) -> std::result::Result<(), Box> { let tmp_src = tempfile::tempdir_in(".")?; let tmp_src_path = tmp_src.path(); @@ -133,7 +120,9 @@ pub fn load_dump( let dst_dir = dst_path .as_ref() .parent() - .with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?; + // TODO + //.with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?; + .unwrap(); let tmp_dst = tempfile::tempdir_in(dst_dir)?; @@ -175,7 +164,7 @@ where U: UuidResolverHandle + Send + Sync + Clone + 'static, P: UpdateActorHandle + Send + Sync + Clone + 'static, { - async fn run(self) -> anyhow::Result<()> { + async fn run(self) -> Result<()> { info!("Performing dump."); create_dir_all(&self.path).await?; @@ -196,9 +185,10 @@ where .dump(uuids, temp_dump_path.clone()) .await?; - let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { + let dump_path = tokio::task::spawn_blocking(move || -> Result { let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; - compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; + compression::to_tar_gz(temp_dump_path, temp_dump_file.path()) + .map_err(|e| DumpActorError::Internal(e))?; let dump_path = self.path.join(self.uid).with_extension("dump"); temp_dump_file.persist(&dump_path)?; diff --git a/meilisearch-http/src/index_controller/error.rs b/meilisearch-http/src/index_controller/error.rs new file mode 100644 index 000000000..2931981ba --- /dev/null +++ b/meilisearch-http/src/index_controller/error.rs @@ -0,0 +1,40 @@ +use std::error::Error; + +use meilisearch_error::Code; +use meilisearch_error::ErrorCode; + +use super::dump_actor::error::DumpActorError; +use super::index_actor::error::IndexActorError; +use super::update_actor::error::UpdateActorError; +use super::uuid_resolver::UuidResolverError; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum IndexControllerError { + #[error("Internal error: {0}")] + Internal(Box), + #[error("Missing index uid")] + MissingUid, + #[error("error resolving index uid: {0}")] + Uuid(#[from] UuidResolverError), + #[error("error with index: {0}")] + IndexActor(#[from] IndexActorError), + #[error("error with update: {0}")] + UpdateActor(#[from] UpdateActorError), + #[error("error with dump: {0}")] + DumpActor(#[from] DumpActorError), +} + +impl ErrorCode for IndexControllerError { + fn error_code(&self) -> Code { + match self { + IndexControllerError::Internal(_) => Code::Internal, + IndexControllerError::MissingUid => Code::InvalidIndexUid, + IndexControllerError::Uuid(e) => e.error_code(), + IndexControllerError::IndexActor(e) => e.error_code(), + IndexControllerError::UpdateActor(e) => e.error_code(), + IndexControllerError::DumpActor(e) => e.error_code(), + } + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index c35e685a8..c714cfd25 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -19,7 +19,8 @@ use crate::index_controller::{ }; use crate::option::IndexerOpts; -use super::{IndexError, IndexMeta, IndexMsg, IndexResult, IndexSettings, IndexStore}; +use super::{IndexMeta, IndexMsg, IndexSettings, IndexStore}; +use super::error::{Result, IndexActorError}; pub const CONCURRENT_INDEX_MSG: usize = 10; @@ -30,7 +31,7 @@ pub struct IndexActor { } impl IndexActor { - pub fn new(receiver: mpsc::Receiver, store: S) -> IndexResult { + pub fn new(receiver: mpsc::Receiver, store: S) -> std::result::Result> { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options)?; let update_handler = Arc::new(update_handler); @@ -137,20 +138,22 @@ impl IndexActor { } } - async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> anyhow::Result { + async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> Result { let index = self .store .get(uuid) .await? - .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || index.perform_search(query)).await? + .ok_or(IndexActorError::UnexistingIndex)?; + let result = spawn_blocking(move || index.perform_search(query)).await??; + Ok(result) + } async fn handle_create_index( &self, uuid: Uuid, primary_key: Option, - ) -> IndexResult { + ) -> Result { let index = self.store.create(uuid, primary_key).await?; let meta = spawn_blocking(move || IndexMeta::new(&index)).await??; Ok(meta) @@ -161,7 +164,7 @@ impl IndexActor { uuid: Uuid, meta: Processing, data: Option, - ) -> IndexResult> { + ) -> Result> { debug!("Processing update {}", meta.id()); let update_handler = self.update_handler.clone(); let index = match self.store.get(uuid).await? { @@ -172,12 +175,12 @@ impl IndexActor { Ok(spawn_blocking(move || update_handler.handle_update(meta, data, index)).await?) } - async fn handle_settings(&self, uuid: Uuid) -> IndexResult> { + async fn handle_settings(&self, uuid: Uuid) -> Result> { let index = self .store .get(uuid) .await? - .ok_or(IndexError::UnexistingIndex)?; + .ok_or(IndexActorError::UnexistingIndex)?; let result = spawn_blocking(move || index.settings()).await??; Ok(result) } @@ -188,12 +191,12 @@ impl IndexActor { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> IndexResult> { + ) -> Result> { let index = self .store .get(uuid) .await? - .ok_or(IndexError::UnexistingIndex)?; + .ok_or(IndexActorError::UnexistingIndex)?; let result = spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve)) .await??; @@ -206,12 +209,12 @@ impl IndexActor { uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ) -> IndexResult { + ) -> Result { let index = self .store .get(uuid) .await? - .ok_or(IndexError::UnexistingIndex)?; + .ok_or(IndexActorError::UnexistingIndex)?; let result = spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve)) @@ -220,7 +223,7 @@ impl IndexActor { Ok(result) } - async fn handle_delete(&self, uuid: Uuid) -> IndexResult<()> { + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { let index = self.store.delete(uuid).await?; if let Some(index) = index { @@ -237,13 +240,13 @@ impl IndexActor { Ok(()) } - async fn handle_get_meta(&self, uuid: Uuid) -> IndexResult { + async fn handle_get_meta(&self, uuid: Uuid) -> Result { match self.store.get(uuid).await? { Some(index) => { let meta = spawn_blocking(move || IndexMeta::new(&index)).await??; Ok(meta) } - None => Err(IndexError::UnexistingIndex), + None => Err(IndexActorError::UnexistingIndex), } } @@ -251,23 +254,23 @@ impl IndexActor { &self, uuid: Uuid, index_settings: IndexSettings, - ) -> IndexResult { + ) -> Result { let index = self .store .get(uuid) .await? - .ok_or(IndexError::UnexistingIndex)?; + .ok_or(IndexActorError::UnexistingIndex)?; let result = spawn_blocking(move || match index_settings.primary_key { Some(primary_key) => { let mut txn = index.write_txn()?; if index.primary_key(&txn)?.is_some() { - return Err(IndexError::ExistingPrimaryKey); + return Err(IndexActorError::ExistingPrimaryKey); } let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); builder.set_primary_key(primary_key); builder.execute(|_, _| ()) - .map_err(|e| IndexError::Internal(e.to_string()))?; + .map_err(|e| IndexActorError::Internal(Box::new(e)))?; let meta = IndexMeta::new_txn(&index, &txn)?; txn.commit()?; Ok(meta) @@ -282,7 +285,7 @@ impl IndexActor { Ok(result) } - async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> IndexResult<()> { + async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> { use tokio::fs::create_dir_all; path.push("indexes"); @@ -294,7 +297,7 @@ impl IndexActor { create_dir_all(&index_path).await?; index_path.push("data.mdb"); - spawn_blocking(move || -> anyhow::Result<()> { + spawn_blocking(move || -> Result<()> { // Get write txn to wait for ongoing write transaction before snapshot. let _txn = index.write_txn()?; index @@ -310,12 +313,12 @@ impl IndexActor { /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the /// documents and all the settings. - async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let index = self .store .get(uuid) .await? - .ok_or(IndexError::UnexistingIndex)?; + .ok_or(IndexActorError::UnexistingIndex)?; let path = path.join(format!("indexes/index-{}/", uuid)); fs::create_dir_all(&path).await?; @@ -325,12 +328,12 @@ impl IndexActor { Ok(()) } - async fn handle_get_stats(&self, uuid: Uuid) -> IndexResult { + async fn handle_get_stats(&self, uuid: Uuid) -> Result { let index = self .store .get(uuid) .await? - .ok_or(IndexError::UnexistingIndex)?; + .ok_or(IndexActorError::UnexistingIndex)?; spawn_blocking(move || { let rtxn = index.read_txn()?; @@ -338,9 +341,10 @@ impl IndexActor { Ok(IndexStats { size: index.size(), number_of_documents: index.number_of_documents(&rtxn) - .map_err(|e| IndexError::Internal(e.to_string()))?, + .map_err(|e| IndexActorError::Internal(Box::new(e)))?, is_indexing: None, - fields_distribution: index.fields_distribution(&rtxn)?, + fields_distribution: index.fields_distribution(&rtxn) + .map_err(|e| IndexActorError::Internal(e.into()))?, }) }) .await? diff --git a/meilisearch-http/src/index_controller/index_actor/error.rs b/meilisearch-http/src/index_controller/index_actor/error.rs new file mode 100644 index 000000000..a124d8b1e --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/error.rs @@ -0,0 +1,49 @@ +use meilisearch_error::{Code, ErrorCode}; + +use crate::index::error::IndexError; + +pub type Result = std::result::Result; + +#[derive(thiserror::Error, Debug)] +pub enum IndexActorError { + #[error("index error: {0}")] + IndexError(#[from] IndexError), + #[error("index already exists")] + IndexAlreadyExists, + #[error("Index doesn't exists")] + UnexistingIndex, + #[error("Existing primary key")] + ExistingPrimaryKey, + #[error("Internal Index Error: {0}")] + Internal(Box), +} + +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for IndexActorError { + fn from(other: $other) -> Self { + Self::Internal(Box::new(other)) + } + } + )* + } +} + +internal_error!( + heed::Error, + tokio::task::JoinError, + std::io::Error +); + +impl ErrorCode for IndexActorError { + fn error_code(&self) -> Code { + match self { + IndexActorError::IndexError(e) => e.error_code(), + IndexActorError::IndexAlreadyExists => Code::IndexAlreadyExists, + IndexActorError::UnexistingIndex => Code::IndexNotFound, + IndexActorError::ExistingPrimaryKey => Code::PrimaryKeyAlreadyPresent, + IndexActorError::Internal(_) => Code::Internal, + } + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 6bf83c647..6afd409d2 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -12,7 +12,8 @@ use crate::{ index_controller::{Failed, Processed}, }; -use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, IndexResult, MapIndexStore}; +use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore}; +use super::error::Result; #[derive(Clone)] pub struct IndexActorHandleImpl { @@ -25,7 +26,7 @@ impl IndexActorHandle for IndexActorHandleImpl { &self, uuid: Uuid, primary_key: Option, - ) -> IndexResult { + ) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { ret, @@ -41,7 +42,7 @@ impl IndexActorHandle for IndexActorHandleImpl { uuid: Uuid, meta: Processing, data: Option, - ) -> anyhow::Result> { + ) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Update { ret, @@ -53,14 +54,14 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn search(&self, uuid: Uuid, query: SearchQuery) -> IndexResult { + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Search { uuid, query, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn settings(&self, uuid: Uuid) -> IndexResult> { + async fn settings(&self, uuid: Uuid) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Settings { uuid, ret }; let _ = self.sender.send(msg).await; @@ -73,7 +74,7 @@ impl IndexActorHandle for IndexActorHandleImpl { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> IndexResult> { + ) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Documents { uuid, @@ -91,7 +92,7 @@ impl IndexActorHandle for IndexActorHandleImpl { uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ) -> IndexResult { + ) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Document { uuid, @@ -103,14 +104,14 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn delete(&self, uuid: Uuid) -> IndexResult<()> { + async fn delete(&self, uuid: Uuid) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Delete { uuid, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn get_index_meta(&self, uuid: Uuid) -> IndexResult { + async fn get_index_meta(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetMeta { uuid, ret }; let _ = self.sender.send(msg).await; @@ -121,7 +122,7 @@ impl IndexActorHandle for IndexActorHandleImpl { &self, uuid: Uuid, index_settings: IndexSettings, - ) -> IndexResult { + ) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::UpdateIndex { uuid, @@ -132,21 +133,21 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Snapshot { uuid, path, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Dump { uuid, path, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { + async fn get_index_stats(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetStats { uuid, ret }; let _ = self.sender.send(msg).await; @@ -155,7 +156,7 @@ impl IndexActorHandle for IndexActorHandleImpl { } impl IndexActorHandleImpl { - pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { + pub fn new(path: impl AsRef, index_size: usize) -> std::result::Result> { let (sender, receiver) = mpsc::channel(100); let store = MapIndexStore::new(path, index_size); diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index e7304d56c..d9c0dabd9 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -5,8 +5,9 @@ use uuid::Uuid; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::index_controller::{Failed, IndexStats, Processed, Processing}; +use super::error::Result as IndexResult; -use super::{IndexMeta, IndexResult, IndexSettings}; +use super::{IndexMeta, IndexSettings}; #[allow(clippy::large_enum_variant)] pub enum IndexMsg { @@ -24,7 +25,7 @@ pub enum IndexMsg { Search { uuid: Uuid, query: SearchQuery, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, Settings { uuid: Uuid, diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index b54a676b0..a66f1b9ea 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -5,7 +5,6 @@ use chrono::{DateTime, Utc}; #[cfg(test)] use mockall::automock; use serde::{Deserialize, Serialize}; -use thiserror::Error; use uuid::Uuid; use actor::IndexActor; @@ -16,6 +15,9 @@ use store::{IndexStore, MapIndexStore}; use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings}; use crate::index_controller::{Failed, IndexStats, Processed, Processing}; +use error::Result; + +use self::error::IndexActorError; use super::IndexSettings; @@ -23,8 +25,7 @@ mod actor; mod handle_impl; mod message; mod store; - -pub type IndexResult = std::result::Result; +pub mod error; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -35,18 +36,18 @@ pub struct IndexMeta { } impl IndexMeta { - fn new(index: &Index) -> IndexResult { + fn new(index: &Index) -> Result { let txn = index.read_txn()?; Self::new_txn(index, &txn) } - fn new_txn(index: &Index, txn: &heed::RoTxn) -> IndexResult { + fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { let created_at = index .created_at(&txn) - .map_err(|e| IndexError::Internal(e.to_string()))?; + .map_err(|e| IndexActorError::Internal(Box::new(e)))?; let updated_at = index .updated_at(&txn) - .map_err(|e| IndexError::Internal(e.to_string()))?; + .map_err(|e| IndexActorError::Internal(Box::new(e)))?; let primary_key = index.primary_key(&txn)?.map(String::from); Ok(Self { created_at, @@ -56,50 +57,19 @@ impl IndexMeta { } } -#[derive(Error, Debug)] -pub enum IndexError { - #[error("index already exists")] - IndexAlreadyExists, - #[error("Index doesn't exists")] - UnexistingIndex, - #[error("Existing primary key")] - ExistingPrimaryKey, - #[error("Internal Index Error: {0}")] - Internal(String), -} - -macro_rules! internal_error { - ($($other:path), *) => { - $( - impl From<$other> for IndexError { - fn from(other: $other) -> Self { - Self::Internal(other.to_string()) - } - } - )* - } -} - -internal_error!( - anyhow::Error, - heed::Error, - tokio::task::JoinError, - std::io::Error -); - #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait IndexActorHandle { async fn create_index(&self, uuid: Uuid, primary_key: Option) - -> IndexResult; + -> Result; async fn update( &self, uuid: Uuid, meta: Processing, data: Option, - ) -> anyhow::Result>; - async fn search(&self, uuid: Uuid, query: SearchQuery) -> IndexResult; - async fn settings(&self, uuid: Uuid) -> IndexResult>; + ) -> Result>; + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result; + async fn settings(&self, uuid: Uuid) -> Result>; async fn documents( &self, @@ -107,23 +77,23 @@ pub trait IndexActorHandle { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> IndexResult>; + ) -> Result>; async fn document( &self, uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ) -> IndexResult; - async fn delete(&self, uuid: Uuid) -> IndexResult<()>; - async fn get_index_meta(&self, uuid: Uuid) -> IndexResult; + ) -> Result; + async fn delete(&self, uuid: Uuid) -> Result<()>; + async fn get_index_meta(&self, uuid: Uuid) -> Result; async fn update_index( &self, uuid: Uuid, index_settings: IndexSettings, - ) -> IndexResult; - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; - async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; - async fn get_index_stats(&self, uuid: Uuid) -> IndexResult; + ) -> Result; + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn get_index_stats(&self, uuid: Uuid) -> Result; } #[cfg(test)] @@ -139,7 +109,7 @@ mod test { &self, uuid: Uuid, primary_key: Option, - ) -> IndexResult { + ) -> Result { self.as_ref().create_index(uuid, primary_key).await } @@ -148,15 +118,15 @@ mod test { uuid: Uuid, meta: Processing, data: Option, - ) -> anyhow::Result> { + ) -> Result> { self.as_ref().update(uuid, meta, data).await } - async fn search(&self, uuid: Uuid, query: SearchQuery) -> IndexResult { + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { self.as_ref().search(uuid, query).await } - async fn settings(&self, uuid: Uuid) -> IndexResult> { + async fn settings(&self, uuid: Uuid) -> Result> { self.as_ref().settings(uuid).await } @@ -166,7 +136,7 @@ mod test { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> IndexResult> { + ) -> Result> { self.as_ref() .documents(uuid, offset, limit, attributes_to_retrieve) .await @@ -177,17 +147,17 @@ mod test { uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ) -> IndexResult { + ) -> Result { self.as_ref() .document(uuid, doc_id, attributes_to_retrieve) .await } - async fn delete(&self, uuid: Uuid) -> IndexResult<()> { + async fn delete(&self, uuid: Uuid) -> Result<()> { self.as_ref().delete(uuid).await } - async fn get_index_meta(&self, uuid: Uuid) -> IndexResult { + async fn get_index_meta(&self, uuid: Uuid) -> Result { self.as_ref().get_index_meta(uuid).await } @@ -195,19 +165,19 @@ mod test { &self, uuid: Uuid, index_settings: IndexSettings, - ) -> IndexResult { + ) -> Result { self.as_ref().update_index(uuid, index_settings).await } - async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { self.as_ref().snapshot(uuid, path).await } - async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { self.as_ref().dump(uuid, path).await } - async fn get_index_stats(&self, uuid: Uuid) -> IndexResult { + async fn get_index_stats(&self, uuid: Uuid) -> Result { self.as_ref().get_index_stats(uuid).await } } diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 39a6e64a6..4c2aed622 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -8,16 +8,16 @@ use tokio::sync::RwLock; use tokio::task::spawn_blocking; use uuid::Uuid; -use super::{IndexError, IndexResult}; +use super::error::{IndexActorError, Result}; use crate::index::Index; type AsyncMap = Arc>>; #[async_trait::async_trait] pub trait IndexStore { - async fn create(&self, uuid: Uuid, primary_key: Option) -> IndexResult; - async fn get(&self, uuid: Uuid) -> IndexResult>; - async fn delete(&self, uuid: Uuid) -> IndexResult>; + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn get(&self, uuid: Uuid) -> Result>; + async fn delete(&self, uuid: Uuid) -> Result>; } pub struct MapIndexStore { @@ -40,7 +40,7 @@ impl MapIndexStore { #[async_trait::async_trait] impl IndexStore for MapIndexStore { - async fn create(&self, uuid: Uuid, primary_key: Option) -> IndexResult { + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result { // We need to keep the lock until we are sure the db file has been opened correclty, to // ensure that another db is not created at the same time. let mut lock = self.index_store.write().await; @@ -50,11 +50,11 @@ impl IndexStore for MapIndexStore { } let path = self.path.join(format!("index-{}", uuid)); if path.exists() { - return Err(IndexError::IndexAlreadyExists); + return Err(IndexActorError::IndexAlreadyExists); } let index_size = self.index_size; - let index = spawn_blocking(move || -> IndexResult { + let index = spawn_blocking(move || -> Result { let index = Index::open(path, index_size)?; if let Some(primary_key) = primary_key { let mut txn = index.write_txn()?; @@ -62,7 +62,7 @@ impl IndexStore for MapIndexStore { let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); builder.set_primary_key(primary_key); builder.execute(|_, _| ()) - .map_err(|e| IndexError::Internal(e.to_string()))?; + .map_err(|e| IndexActorError::Internal(Box::new(e)))?; txn.commit()?; } @@ -75,7 +75,7 @@ impl IndexStore for MapIndexStore { Ok(index) } - async fn get(&self, uuid: Uuid) -> IndexResult> { + async fn get(&self, uuid: Uuid) -> Result> { let guard = self.index_store.read().await; match guard.get(&uuid) { Some(index) => Ok(Some(index.clone())), @@ -95,7 +95,7 @@ impl IndexStore for MapIndexStore { } } - async fn delete(&self, uuid: Uuid) -> IndexResult> { + async fn delete(&self, uuid: Uuid) -> Result> { let db_path = self.path.join(format!("index-{}", uuid)); fs::remove_dir_all(db_path).await?; let index = self.index_store.write().await.remove(&uuid); diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 0c801558b..0046bd6a5 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use std::time::Duration; use actix_web::web::{Bytes, Payload}; -use anyhow::bail; use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use log::info; @@ -24,8 +23,10 @@ use uuid_resolver::{UuidResolverError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; +use error::Result; use self::dump_actor::load_dump; +use self::error::IndexControllerError; mod dump_actor; mod index_actor; @@ -33,6 +34,7 @@ mod snapshot; mod update_actor; mod updates; mod uuid_resolver; +pub mod error; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -81,7 +83,7 @@ pub struct Stats { } impl IndexController { - pub fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { + pub fn new(path: impl AsRef, options: &Opt) -> std::result::Result> { let index_size = options.max_mdb_size.get_bytes() as usize; let update_store_size = options.max_udb_size.get_bytes() as usize; @@ -151,7 +153,7 @@ impl IndexController { format: milli::update::UpdateFormat, payload: Payload, primary_key: Option, - ) -> anyhow::Result { + ) -> Result { let perform_update = |uuid| async move { let meta = UpdateMeta::DocumentsAddition { method, @@ -189,7 +191,7 @@ impl IndexController { } } - pub async fn clear_documents(&self, uid: String) -> anyhow::Result { + pub async fn clear_documents(&self, uid: String) -> Result { let uuid = self.uuid_resolver.get(uid).await?; let meta = UpdateMeta::ClearDocuments; let (_, receiver) = mpsc::channel(1); @@ -201,7 +203,7 @@ impl IndexController { &self, uid: String, documents: Vec, - ) -> anyhow::Result { + ) -> Result { let uuid = self.uuid_resolver.get(uid).await?; let meta = UpdateMeta::DeleteDocuments { ids: documents }; let (_, receiver) = mpsc::channel(1); @@ -214,7 +216,7 @@ impl IndexController { uid: String, settings: Settings, create: bool, - ) -> anyhow::Result { + ) -> Result { let perform_udpate = |uuid| async move { let meta = UpdateMeta::Settings(settings.into_unchecked()); // Nothing so send, drop the sender right away, as not to block the update actor. @@ -239,9 +241,9 @@ impl IndexController { pub async fn create_index( &self, index_settings: IndexSettings, - ) -> anyhow::Result { + ) -> Result { let IndexSettings { uid, primary_key } = index_settings; - let uid = uid.ok_or_else(|| anyhow::anyhow!("Can't create an index without a uid."))?; + let uid = uid.ok_or(IndexControllerError::MissingUid)?; let uuid = Uuid::new_v4(); let meta = self.index_handle.create_index(uuid, primary_key).await?; self.uuid_resolver.insert(uid.clone(), uuid).await?; @@ -255,26 +257,26 @@ impl IndexController { Ok(meta) } - pub async fn delete_index(&self, uid: String) -> anyhow::Result<()> { + pub async fn delete_index(&self, uid: String) -> Result<()> { let uuid = self.uuid_resolver.delete(uid).await?; self.update_handle.delete(uuid).await?; self.index_handle.delete(uuid).await?; Ok(()) } - pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result { + pub async fn update_status(&self, uid: String, id: u64) -> Result { let uuid = self.uuid_resolver.get(uid).await?; let result = self.update_handle.update_status(uuid, id).await?; Ok(result) } - pub async fn all_update_status(&self, uid: String) -> anyhow::Result> { + pub async fn all_update_status(&self, uid: String) -> Result> { let uuid = self.uuid_resolver.get(uid).await?; let result = self.update_handle.get_all_updates_status(uuid).await?; Ok(result) } - pub async fn list_indexes(&self) -> anyhow::Result> { + pub async fn list_indexes(&self) -> Result> { let uuids = self.uuid_resolver.list().await?; let mut ret = Vec::new(); @@ -293,7 +295,7 @@ impl IndexController { Ok(ret) } - pub async fn settings(&self, uid: String) -> anyhow::Result> { + pub async fn settings(&self, uid: String) -> Result> { let uuid = self.uuid_resolver.get(uid.clone()).await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) @@ -305,7 +307,7 @@ impl IndexController { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> anyhow::Result> { + ) -> Result> { let uuid = self.uuid_resolver.get(uid.clone()).await?; let documents = self .index_handle @@ -319,7 +321,7 @@ impl IndexController { uid: String, doc_id: String, attributes_to_retrieve: Option>, - ) -> anyhow::Result { + ) -> Result { let uuid = self.uuid_resolver.get(uid.clone()).await?; let document = self .index_handle @@ -332,9 +334,9 @@ impl IndexController { &self, uid: String, index_settings: IndexSettings, - ) -> anyhow::Result { + ) -> Result { if index_settings.uid.is_some() { - bail!("Can't change the index uid.") + todo!("Can't change the index uid.") } let uuid = self.uuid_resolver.get(uid.clone()).await?; @@ -348,13 +350,13 @@ impl IndexController { Ok(meta) } - pub async fn search(&self, uid: String, query: SearchQuery) -> anyhow::Result { + pub async fn search(&self, uid: String, query: SearchQuery) -> Result { let uuid = self.uuid_resolver.get(uid).await?; let result = self.index_handle.search(uuid, query).await?; Ok(result) } - pub async fn get_index(&self, uid: String) -> anyhow::Result { + pub async fn get_index(&self, uid: String) -> Result { let uuid = self.uuid_resolver.get(uid.clone()).await?; let meta = self.index_handle.get_index_meta(uuid).await?; let meta = IndexMetadata { @@ -366,11 +368,11 @@ impl IndexController { Ok(meta) } - pub async fn get_uuids_size(&self) -> anyhow::Result { + pub async fn get_uuids_size(&self) -> Result { Ok(self.uuid_resolver.get_size().await?) } - pub async fn get_index_stats(&self, uid: String) -> anyhow::Result { + pub async fn get_index_stats(&self, uid: String) -> Result { let uuid = self.uuid_resolver.get(uid).await?; let update_infos = self.update_handle.get_info().await?; let mut stats = self.index_handle.get_index_stats(uuid).await?; @@ -379,7 +381,7 @@ impl IndexController { Ok(stats) } - pub async fn get_all_stats(&self) -> anyhow::Result { + pub async fn get_all_stats(&self) -> Result { let update_infos = self.update_handle.get_info().await?; let mut database_size = self.get_uuids_size().await? + update_infos.size; let mut last_update: Option> = None; @@ -405,11 +407,11 @@ impl IndexController { }) } - pub async fn create_dump(&self) -> anyhow::Result { + pub async fn create_dump(&self) -> Result { Ok(self.dump_handle.create_dump().await?) } - pub async fn dump_info(&self, uid: String) -> anyhow::Result { + pub async fn dump_info(&self, uid: String) -> Result { Ok(self.dump_handle.dump_info(uid).await?) } } diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index daef7d582..171c08eee 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -1,7 +1,6 @@ use std::path::{Path, PathBuf}; use std::time::Duration; -use anyhow::bail; use log::{error, info}; use tokio::fs; use tokio::task::spawn_blocking; @@ -53,7 +52,7 @@ where } } - async fn perform_snapshot(&self) -> anyhow::Result<()> { + async fn perform_snapshot(&self) -> std::result::Result<(), Box> { info!("Performing snapshot."); let snapshot_dir = self.snapshot_path.clone(); @@ -78,7 +77,7 @@ where let snapshot_path = self .snapshot_path .join(format!("{}.snapshot", self.db_name)); - let snapshot_path = spawn_blocking(move || -> anyhow::Result { + let snapshot_path = spawn_blocking(move || -> Result> { let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?; let temp_snapshot_file_path = temp_snapshot_file.path().to_owned(); compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?; @@ -98,7 +97,7 @@ pub fn load_snapshot( snapshot_path: impl AsRef, ignore_snapshot_if_db_exists: bool, ignore_missing_snapshot: bool, -) -> anyhow::Result<()> { +) -> std::result::Result<(), Box> { if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() { match compression::from_tar_gz(snapshot_path, &db_path) { Ok(()) => Ok(()), @@ -109,7 +108,7 @@ pub fn load_snapshot( } } } else if db_path.as_ref().exists() && !ignore_snapshot_if_db_exists { - bail!( + todo!( "database already exists at {:?}, try to delete it or rename it", db_path .as_ref() @@ -117,7 +116,7 @@ pub fn load_snapshot( .unwrap_or_else(|_| db_path.as_ref().to_owned()) ) } else if !snapshot_path.as_ref().exists() && !ignore_missing_snapshot { - bail!( + todo!( "snapshot doesn't exist at {:?}", snapshot_path .as_ref() @@ -142,7 +141,7 @@ mod test { use super::*; use crate::index_controller::index_actor::MockIndexActorHandle; use crate::index_controller::update_actor::{ - MockUpdateActorHandle, UpdateActorHandleImpl, UpdateError, + MockUpdateActorHandle, UpdateActorHandleImpl, error::UpdateActorError, }; use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidResolverError}; @@ -224,7 +223,7 @@ mod test { update_handle .expect_snapshot() // abitrary error - .returning(|_, _| Box::pin(err(UpdateError::UnexistingUpdate(0)))); + .returning(|_, _| Box::pin(err(UpdateActorError::UnexistingUpdate(0)))); let snapshot_path = tempfile::tempdir_in(".").unwrap(); let snapshot_service = SnapshotService::new( diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index eebbf6247..fa8251a0c 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -13,7 +13,8 @@ use tokio::io::AsyncWriteExt; use tokio::sync::mpsc; use uuid::Uuid; -use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStore, UpdateStoreInfo}; +use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo}; +use super::error::{Result, UpdateActorError}; use crate::index_controller::index_actor::IndexActorHandle; use crate::index_controller::{UpdateMeta, UpdateStatus}; @@ -35,7 +36,7 @@ where inbox: mpsc::Receiver>, path: impl AsRef, index_handle: I, - ) -> anyhow::Result { + ) -> std::result::Result> { let path = path.as_ref().join("updates"); std::fs::create_dir_all(&path)?; @@ -202,7 +203,7 @@ where tokio::task::spawn_blocking(move || { let result = store .meta(uuid, id)? - .ok_or(UpdateError::UnexistingUpdate(id))?; + .ok_or(UpdateActorError::UnexistingUpdate(id))?; Ok(result) }) .await? @@ -230,7 +231,7 @@ where let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); - tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + tokio::task::spawn_blocking(move || -> Result<()> { update_store.dump(&uuids, path.to_path_buf(), index_handle)?; Ok(()) }) @@ -241,7 +242,7 @@ where async fn handle_get_info(&self) -> Result { let update_store = self.store.clone(); - let info = tokio::task::spawn_blocking(move || -> anyhow::Result { + let info = tokio::task::spawn_blocking(move || -> Result { let info = update_store.get_info()?; Ok(info) }) diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs new file mode 100644 index 000000000..ee68dd997 --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -0,0 +1,64 @@ +use std::error::Error; + +use meilisearch_error::{Code, ErrorCode}; + +use crate::index_controller::index_actor::error::IndexActorError; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum UpdateActorError { + #[error("Update {0} doesn't exist.")] + UnexistingUpdate(u64), + #[error("Internal error processing update: {0}")] + Internal(Box), + #[error("error with index: {0}")] + IndexActor(#[from] IndexActorError), + #[error( + "Update store was shut down due to a fatal error, please check your logs for more info." + )] + FatalUpdateStoreError, +} + +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for UpdateActorError { + fn from(other: $other) -> Self { + Self::Internal(Box::new(other)) + } + } + )* + } +} + +impl From> for UpdateActorError { + fn from(_: tokio::sync::mpsc::error::SendError) -> Self { + Self::FatalUpdateStoreError + } +} + +impl From for UpdateActorError { + fn from(_: tokio::sync::oneshot::error::RecvError) -> Self { + Self::FatalUpdateStoreError + } +} + +internal_error!( + heed::Error, + std::io::Error, + serde_json::Error, + actix_http::error::PayloadError, + tokio::task::JoinError +); + +impl ErrorCode for UpdateActorError { + fn error_code(&self) -> Code { + match self { + UpdateActorError::UnexistingUpdate(_) => Code::NotFound, + UpdateActorError::Internal(_) => Code::Internal, + UpdateActorError::IndexActor(e) => e.error_code(), + UpdateActorError::FatalUpdateStoreError => Code::Internal, + } + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 7844bf855..f5331554b 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -6,10 +6,8 @@ use uuid::Uuid; use crate::index_controller::{IndexActorHandle, UpdateStatus}; -use super::{ - PayloadData, Result, UpdateActor, UpdateActorHandle, UpdateError, UpdateMeta, UpdateMsg, - UpdateStoreInfo, -}; +use super::error::Result; +use super::{PayloadData, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStoreInfo}; #[derive(Clone)] pub struct UpdateActorHandleImpl { @@ -24,7 +22,7 @@ where index_handle: I, path: impl AsRef, update_store_size: usize, - ) -> anyhow::Result + ) -> std::result::Result> where I: IndexActorHandle + Clone + Send + Sync + 'static, { @@ -48,72 +46,42 @@ where async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::ListUpdates { uuid, ret }; - self.sender - .send(msg) - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)?; - receiver - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)? + self.sender.send(msg).await?; + receiver.await? } async fn update_status(&self, uuid: Uuid, id: u64) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::GetUpdate { uuid, id, ret }; - self.sender - .send(msg) - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)?; - receiver - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)? + self.sender.send(msg).await?; + receiver.await? } async fn delete(&self, uuid: Uuid) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Delete { uuid, ret }; - self.sender - .send(msg) - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)?; - receiver - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)? + self.sender.send(msg).await?; + receiver.await? } async fn snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Snapshot { uuids, path, ret }; - self.sender - .send(msg) - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)?; - receiver - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)? + self.sender.send(msg).await?; + receiver.await? } async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Dump { uuids, path, ret }; - self.sender - .send(msg) - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)?; - receiver - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)? + self.sender.send(msg).await?; + receiver.await? } async fn get_info(&self) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::GetInfo { ret }; - self.sender - .send(msg) - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)?; - receiver - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)? + self.sender.send(msg).await?; + receiver.await? } async fn update( @@ -129,12 +97,7 @@ where meta, ret, }; - self.sender - .send(msg) - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)?; - receiver - .await - .map_err(|_| UpdateError::FatalUpdateStoreError)? + self.sender.send(msg).await?; + receiver.await? } } diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 37df2af32..48a88b097 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -4,7 +4,8 @@ use std::path::PathBuf; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use super::{PayloadData, Result, UpdateMeta, UpdateStatus, UpdateStoreInfo}; +use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo}; +use super::error::Result; pub enum UpdateMsg { Update { diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index b854cca70..42be99cf8 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -1,12 +1,6 @@ -mod actor; -mod handle_impl; -mod message; -pub mod store; - use std::{collections::HashSet, path::PathBuf}; use actix_http::error::PayloadError; -use thiserror::Error; use tokio::sync::mpsc; use uuid::Uuid; @@ -14,49 +8,22 @@ use crate::index_controller::{UpdateMeta, UpdateStatus}; use actor::UpdateActor; use message::UpdateMsg; +use error::Result; pub use handle_impl::UpdateActorHandleImpl; pub use store::{UpdateStore, UpdateStoreInfo}; -pub type Result = std::result::Result; +mod actor; +mod handle_impl; +mod message; +pub mod error; +pub mod store; + type PayloadData = std::result::Result; #[cfg(test)] use mockall::automock; -#[derive(Debug, Error)] -pub enum UpdateError { - #[error("Update {0} doesn't exist.")] - UnexistingUpdate(u64), - #[error("Internal error processing update: {0}")] - Internal(String), - #[error( - "Update store was shut down due to a fatal error, please check your logs for more info." - )] - FatalUpdateStoreError, -} - -macro_rules! internal_error { - ($($other:path), *) => { - $( - impl From<$other> for UpdateError { - fn from(other: $other) -> Self { - Self::Internal(other.to_string()) - } - } - )* - } -} - -internal_error!( - heed::Error, - std::io::Error, - serde_json::Error, - PayloadError, - tokio::task::JoinError, - anyhow::Error -); - #[async_trait::async_trait] #[cfg_attr(test, automock(type Data=Vec;))] pub trait UpdateActorHandle { diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index e7f36a2a1..a621feebe 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -9,7 +9,7 @@ use heed::{EnvOpenOptions, RoTxn}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use super::{State, UpdateStore}; +use super::{State, UpdateStore, Result}; use crate::index_controller::{ index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued, UpdateStatus, @@ -27,7 +27,7 @@ impl UpdateStore { uuids: &HashSet, path: PathBuf, handle: impl IndexActorHandle, - ) -> anyhow::Result<()> { + ) -> Result<()> { let state_lock = self.state.write(); state_lock.swap(State::Dumping); @@ -52,7 +52,7 @@ impl UpdateStore { txn: &RoTxn, uuids: &HashSet, path: impl AsRef, - ) -> anyhow::Result<()> { + ) -> Result<()> { let dump_data_path = path.as_ref().join("data.jsonl"); let mut dump_data_file = File::create(dump_data_path)?; @@ -71,7 +71,7 @@ impl UpdateStore { uuids: &HashSet, mut file: &mut File, dst_path: impl AsRef, - ) -> anyhow::Result<()> { + ) -> Result<()> { let pendings = self.pending_queue.iter(txn)?.lazily_decode_data(); for pending in pendings { @@ -103,7 +103,7 @@ impl UpdateStore { txn: &RoTxn, uuids: &HashSet, mut file: &mut File, - ) -> anyhow::Result<()> { + ) -> Result<()> { let updates = self.updates.iter(txn)?.lazily_decode_data(); for update in updates { @@ -125,7 +125,7 @@ impl UpdateStore { src: impl AsRef, dst: impl AsRef, db_size: usize, - ) -> anyhow::Result<()> { + ) -> std::result::Result<(), Box> { let dst_update_path = dst.as_ref().join("updates/"); create_dir_all(&dst_update_path)?; @@ -175,7 +175,7 @@ async fn dump_indexes( uuids: &HashSet, handle: impl IndexActorHandle, path: impl AsRef, -) -> anyhow::Result<()> { +) -> Result<()> { for uuid in uuids { handle.dump(*uuid, path.as_ref().to_owned()).await?; } diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index e7b719fc9..429ce362b 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -24,8 +24,9 @@ use uuid::Uuid; use codec::*; use super::UpdateMeta; +use super::error::Result; use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle}; -use crate::{helpers::EnvSizer, index_controller::index_actor::IndexResult}; +use crate::helpers::EnvSizer; #[allow(clippy::upper_case_acronyms)] type BEU64 = U64; @@ -109,7 +110,7 @@ impl UpdateStore { fn new( mut options: EnvOpenOptions, path: impl AsRef, - ) -> anyhow::Result<(Self, mpsc::Receiver<()>)> { + ) -> std::result::Result<(Self, mpsc::Receiver<()>), Box> { options.max_dbs(5); let env = options.open(&path)?; @@ -140,7 +141,7 @@ impl UpdateStore { path: impl AsRef, index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static, must_exit: Arc, - ) -> anyhow::Result> { + ) -> std::result::Result, Box> { let (update_store, mut notification_receiver) = Self::new(options, path)?; let update_store = Arc::new(update_store); @@ -285,7 +286,7 @@ impl UpdateStore { fn process_pending_update( &self, index_handle: impl IndexActorHandle, - ) -> anyhow::Result> { + ) -> Result> { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; let first_meta = self.pending_queue.first(&rtxn)?; @@ -320,7 +321,7 @@ impl UpdateStore { index_handle: impl IndexActorHandle, index_uuid: Uuid, global_id: u64, - ) -> anyhow::Result> { + ) -> Result> { let content_path = content.map(|uuid| update_uuid_to_file_path(&self.path, uuid)); let update_id = processing.id(); @@ -368,7 +369,7 @@ impl UpdateStore { } /// List the updates for `index_uuid`. - pub fn list(&self, index_uuid: Uuid) -> anyhow::Result> { + pub fn list(&self, index_uuid: Uuid) -> Result> { let mut update_list = BTreeMap::::new(); let txn = self.env.read_txn()?; @@ -437,7 +438,7 @@ impl UpdateStore { } /// Delete all updates for an index from the update store. - pub fn delete_all(&self, index_uuid: Uuid) -> anyhow::Result<()> { + pub fn delete_all(&self, index_uuid: Uuid) -> Result<()> { let mut txn = self.env.write_txn()?; // Contains all the content file paths that we need to be removed if the deletion was successful. let mut uuids_to_remove = Vec::new(); @@ -488,7 +489,7 @@ impl UpdateStore { uuids: &HashSet, path: impl AsRef, handle: impl IndexActorHandle + Clone, - ) -> anyhow::Result<()> { + ) -> Result<()> { let state_lock = self.state.write(); state_lock.swap(State::Snapshoting); @@ -535,13 +536,13 @@ impl UpdateStore { while let Some(res) = stream.next().await { res?; } - Ok(()) as IndexResult<()> + Ok(()) as Result<()> })?; Ok(()) } - pub fn get_info(&self) -> anyhow::Result { + pub fn get_info(&self) -> Result { let mut size = self.env.size(); let txn = self.env.read_txn()?; for entry in self.pending_queue.iter(&txn)? { diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index af710dd87..1296264e0 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -12,7 +12,7 @@ pub struct UuidResolverHandleImpl { } impl UuidResolverHandleImpl { - pub fn new(path: impl AsRef) -> anyhow::Result { + pub fn new(path: impl AsRef) -> Result { let (sender, reveiver) = mpsc::channel(100); let store = HeedUuidStore::new(path)?; let actor = UuidResolverActor::new(reveiver, store); @@ -32,7 +32,7 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - async fn delete(&self, name: String) -> anyhow::Result { + async fn delete(&self, name: String) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Delete { uid: name, ret }; let _ = self.sender.send(msg).await; @@ -41,7 +41,7 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - async fn list(&self) -> anyhow::Result> { + async fn list(&self) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::List { ret }; let _ = self.sender.send(msg).await; @@ -50,7 +50,7 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - async fn insert(&self, name: String, uuid: Uuid) -> anyhow::Result<()> { + async fn insert(&self, name: String, uuid: Uuid) -> Result<()> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Insert { ret, name, uuid }; let _ = self.sender.send(msg).await; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 3c3b5fd06..67d77d411 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -6,6 +6,8 @@ pub mod store; use std::collections::HashSet; use std::path::PathBuf; +use meilisearch_error::Code; +use meilisearch_error::ErrorCode; use thiserror::Error; use uuid::Uuid; @@ -27,9 +29,9 @@ pub type Result = std::result::Result; #[cfg_attr(test, automock)] pub trait UuidResolverHandle { async fn get(&self, name: String) -> Result; - async fn insert(&self, name: String, uuid: Uuid) -> anyhow::Result<()>; - async fn delete(&self, name: String) -> anyhow::Result; - async fn list(&self) -> anyhow::Result>; + async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; + async fn delete(&self, name: String) -> Result; + async fn list(&self) -> Result>; async fn snapshot(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; async fn dump(&self, path: PathBuf) -> Result>; @@ -44,7 +46,7 @@ pub enum UuidResolverError { #[error("Badly formatted index uid: {0}")] BadlyFormatted(String), #[error("Internal error resolving index uid: {0}")] - Internal(String), + Internal(Box), } macro_rules! internal_error { @@ -52,7 +54,7 @@ macro_rules! internal_error { $( impl From<$other> for UuidResolverError { fn from(other: $other) -> Self { - Self::Internal(other.to_string()) + Self::Internal(Box::new(other)) } } )* @@ -66,3 +68,14 @@ internal_error!( tokio::task::JoinError, serde_json::Error ); + +impl ErrorCode for UuidResolverError { + fn error_code(&self) -> Code { + match self { + UuidResolverError::NameAlreadyExist => Code::IndexAlreadyExists, + UuidResolverError::UnexistingIndex(_) => Code::IndexNotFound, + UuidResolverError::BadlyFormatted(_) => Code::InvalidIndexUid, + UuidResolverError::Internal(_) => Code::Internal, + } + } +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index bab223bb3..88d707797 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -39,7 +39,7 @@ pub struct HeedUuidStore { } impl HeedUuidStore { - pub fn new(path: impl AsRef) -> anyhow::Result { + pub fn new(path: impl AsRef) -> Result { let path = path.as_ref().join(UUIDS_DB_PATH); create_dir_all(&path)?; let mut options = EnvOpenOptions::new(); @@ -153,7 +153,7 @@ impl HeedUuidStore { Ok(uuids) } - pub fn load_dump(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + pub fn load_dump(src: impl AsRef, dst: impl AsRef) -> Result<()> { let uuid_resolver_path = dst.as_ref().join(UUIDS_DB_PATH); std::fs::create_dir_all(&uuid_resolver_path)?; diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 36f5bdf4d..3ae38bb21 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -31,9 +31,9 @@ pub struct SearchQueryGet { } impl TryFrom for SearchQuery { - type Error = anyhow::Error; + type Error = Box; - fn try_from(other: SearchQueryGet) -> anyhow::Result { + fn try_from(other: SearchQueryGet) -> Result { let attributes_to_retrieve = other .attributes_to_retrieve .map(|attrs| attrs.split(',').map(String::from).collect::>()); From d1550670a8aefe51cf46cc9db003c8cd17f5ed69 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 15 Jun 2021 16:02:49 +0200 Subject: [PATCH 436/522] enable response error for index routes --- meilisearch-http/src/routes/index.rs | 79 ++++++++-------------------- 1 file changed, 22 insertions(+), 57 deletions(-) diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index efe2ef711..137b4dbbd 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -3,9 +3,9 @@ use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use super::{IndexParam, UpdateStatusResponse}; use crate::error::ResponseError; use crate::helpers::Authentication; -use super::{UpdateStatusResponse, IndexParam}; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { @@ -20,12 +20,8 @@ pub fn services(cfg: &mut web::ServiceConfig) { #[get("/indexes", wrap = "Authentication::Private")] async fn list_indexes(data: web::Data) -> Result { - match data.list_indexes().await { - Ok(indexes) => Ok(HttpResponse::Ok().json(indexes)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let indexes = data.list_indexes().await?; + Ok(HttpResponse::Ok().json(indexes)) } #[get("/indexes/{index_uid}", wrap = "Authentication::Private")] @@ -33,12 +29,8 @@ async fn get_index( data: web::Data, path: web::Path, ) -> Result { - match data.index(path.index_uid.clone()).await { - Ok(meta) => Ok(HttpResponse::Ok().json(meta)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let meta = data.index(path.index_uid.clone()).await?; + Ok(HttpResponse::Ok().json(meta)) } #[derive(Debug, Deserialize)] @@ -54,12 +46,8 @@ async fn create_index( body: web::Json, ) -> Result { let body = body.into_inner(); - match data.create_index(body.uid, body.primary_key).await { - Ok(meta) => Ok(HttpResponse::Ok().json(meta)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let meta = data.create_index(body.uid, body.primary_key).await?; + Ok(HttpResponse::Ok().json(meta)) } #[derive(Debug, Deserialize)] @@ -86,15 +74,10 @@ async fn update_index( body: web::Json, ) -> Result { let body = body.into_inner(); - match data + let meta = data .update_index(path.into_inner().index_uid, body.primary_key, body.uid) - .await - { - Ok(meta) => Ok(HttpResponse::Ok().json(meta)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + .await?; + Ok(HttpResponse::Ok().json(meta)) } #[delete("/indexes/{index_uid}", wrap = "Authentication::Private")] @@ -102,12 +85,8 @@ async fn delete_index( data: web::Data, path: web::Path, ) -> Result { - match data.delete_index(path.index_uid.clone()).await { - Ok(_) => Ok(HttpResponse::NoContent().finish()), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + data.delete_index(path.index_uid.clone()).await?; + Ok(HttpResponse::NoContent().finish()) } #[derive(Deserialize)] @@ -125,18 +104,11 @@ async fn get_update_status( path: web::Path, ) -> Result { let params = path.into_inner(); - let result = data + let meta = data .get_update_status(params.index_uid, params.update_id) - .await; - match result { - Ok(meta) => { - let meta = UpdateStatusResponse::from(meta); - Ok(HttpResponse::Ok().json(meta)) - }, - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + .await?; + let meta = UpdateStatusResponse::from(meta); + Ok(HttpResponse::Ok().json(meta)) } #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] @@ -144,18 +116,11 @@ async fn get_all_updates_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_updates_status(path.into_inner().index_uid).await; - match result { - Ok(metas) => { - let metas = metas - .into_iter() - .map(UpdateStatusResponse::from) - .collect::>(); + let metas = data.get_updates_status(path.into_inner().index_uid).await?; + let metas = metas + .into_iter() + .map(UpdateStatusResponse::from) + .collect::>(); - Ok(HttpResponse::Ok().json(metas)) - }, - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + Ok(HttpResponse::Ok().json(metas)) } From 112cd1787c2ba5db6b1aa5b72f50428c35321284 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 15 Jun 2021 16:07:25 +0200 Subject: [PATCH 437/522] change error message for uuid resolver --- meilisearch-http/src/index_controller/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/error.rs b/meilisearch-http/src/index_controller/error.rs index 2931981ba..23992ba80 100644 --- a/meilisearch-http/src/index_controller/error.rs +++ b/meilisearch-http/src/index_controller/error.rs @@ -16,7 +16,7 @@ pub enum IndexControllerError { Internal(Box), #[error("Missing index uid")] MissingUid, - #[error("error resolving index uid: {0}")] + #[error("index resolution error: {0}")] Uuid(#[from] UuidResolverError), #[error("error with index: {0}")] IndexActor(#[from] IndexActorError), From 5c52a1393ff42b9e6530e0c727aed56e14cffff2 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 15 Jun 2021 16:15:27 +0200 Subject: [PATCH 438/522] enable response error for settings routes --- .../routes/{settings/mod.rs => settings.rs} | 64 +++++-------------- .../routes/settings/distinct_attributes.rs | 36 ----------- .../src/routes/settings/ranking_rules.rs | 23 ------- .../src/routes/settings/synonyms.rs | 43 ------------- 4 files changed, 16 insertions(+), 150 deletions(-) rename meilisearch-http/src/routes/{settings/mod.rs => settings.rs} (66%) delete mode 100644 meilisearch-http/src/routes/settings/distinct_attributes.rs delete mode 100644 meilisearch-http/src/routes/settings/ranking_rules.rs delete mode 100644 meilisearch-http/src/routes/settings/synonyms.rs diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings.rs similarity index 66% rename from meilisearch-http/src/routes/settings/mod.rs rename to meilisearch-http/src/routes/settings.rs index 45d49132c..10ce7292e 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings.rs @@ -26,14 +26,8 @@ macro_rules! make_setting_route { $attr: Some(None), ..Default::default() }; - match data.update_settings(index_uid.into_inner(), settings, false).await { - Ok(update_status) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) - } - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let update_status = data.update_settings(index_uid.into_inner(), settings, false).await?; + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } #[actix_web::post($route, wrap = "Authentication::Private")] @@ -47,14 +41,8 @@ macro_rules! make_setting_route { ..Default::default() }; - match data.update_settings(index_uid.into_inner(), settings, true).await { - Ok(update_status) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) - } - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let update_status = data.update_settings(index_uid.into_inner(), settings, true).await?; + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } #[actix_web::get($route, wrap = "Authentication::Private")] @@ -62,12 +50,8 @@ macro_rules! make_setting_route { data: actix_web::web::Data, index_uid: actix_web::web::Path, ) -> std::result::Result { - match data.settings(index_uid.into_inner()).await { - Ok(settings) => Ok(HttpResponse::Ok().json(settings.$attr)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let settings = data.settings(index_uid.into_inner()).await?; + Ok(HttpResponse::Ok().json(settings.$attr)) } } }; @@ -148,17 +132,11 @@ async fn update_all( body: web::Json>, ) -> Result { let settings = body.into_inner().check(); - match data + let update_result = data .update_settings(index_uid.into_inner(), settings, true) - .await - { - Ok(update_result) => Ok( - HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_result.id() })) - ), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + .await?; + let json = serde_json::json!({ "updateId": update_result.id() }); + Ok(HttpResponse::Accepted().json(json)) } #[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] @@ -166,12 +144,8 @@ async fn get_all( data: web::Data, index_uid: web::Path, ) -> Result { - match data.settings(index_uid.into_inner()).await { - Ok(settings) => Ok(HttpResponse::Ok().json(settings)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let settings = data.settings(index_uid.into_inner()).await?; + Ok(HttpResponse::Ok().json(settings)) } #[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] @@ -180,15 +154,9 @@ async fn delete_all( index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); - match data + let update_result = data .update_settings(index_uid.into_inner(), settings, false) - .await - { - Ok(update_result) => Ok( - HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_result.id() })) - ), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + .await?; + let json = serde_json::json!({ "updateId": update_result.id() }); + Ok(HttpResponse::Accepted().json(json)) } diff --git a/meilisearch-http/src/routes/settings/distinct_attributes.rs b/meilisearch-http/src/routes/settings/distinct_attributes.rs deleted file mode 100644 index 7b991f861..000000000 --- a/meilisearch-http/src/routes/settings/distinct_attributes.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::make_update_delete_routes; -use actix_web::{web, HttpResponse, get}; - -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::Data; - -#[get( - "/indexes/{index_uid}/settings/distinct-attribute", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - index_uid: web::Path, -) -> Result { - let index = data - .db - .load() - .open_index(&index_uid.as_ref()) - .ok_or(Error::index_not_found(&index_uid.as_ref()))?; - let reader = data.db.load().main_read_txn()?; - let distinct_attribute_id = index.main.distinct_attribute(&reader)?; - let schema = index.main.schema(&reader)?; - let distinct_attribute = match (schema, distinct_attribute_id) { - (Some(schema), Some(id)) => schema.name(id).map(str::to_string), - _ => None, - }; - - Ok(HttpResponse::Ok().json(distinct_attribute)) -} - -make_update_delete_routes!( - "/indexes/{index_uid}/settings/distinct-attribute", - String, - distinct_attribute -); diff --git a/meilisearch-http/src/routes/settings/ranking_rules.rs b/meilisearch-http/src/routes/settings/ranking_rules.rs deleted file mode 100644 index e0872954a..000000000 --- a/meilisearch-http/src/routes/settings/ranking_rules.rs +++ /dev/null @@ -1,23 +0,0 @@ -use crate::make_update_delete_routes; -use actix_web::{web, HttpResponse, get}; - -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::Data; - -#[get( - "/indexes/{index_uid}/settings/ranking-rules", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - index_uid: web::Path, -) -> Result { - todo!() -} - -make_update_delete_routes!( - "/indexes/{index_uid}/settings/ranking-rules", - Vec, - ranking_rules -); diff --git a/meilisearch-http/src/routes/settings/synonyms.rs b/meilisearch-http/src/routes/settings/synonyms.rs deleted file mode 100644 index e5b5b2afd..000000000 --- a/meilisearch-http/src/routes/settings/synonyms.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::collections::BTreeMap; - -use actix_web::{web, HttpResponse, get}; -use indexmap::IndexMap; - -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::make_update_delete_routes; -use crate::Data; - -#[get( - "/indexes/{index_uid}/settings/synonyms", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - index_uid: web::Path, -) -> Result { - let index = data - .db - .load() - .open_index(&index_uid.as_ref()) - .ok_or(Error::index_not_found(&index_uid.as_ref()))?; - - let reader = data.db.load().main_read_txn()?; - - let synonyms_list = index.main.synonyms(&reader)?; - - let mut synonyms = IndexMap::new(); - let index_synonyms = &index.synonyms; - for synonym in synonyms_list { - let list = index_synonyms.synonyms(&reader, synonym.as_bytes())?; - synonyms.insert(synonym, list); - } - - Ok(HttpResponse::Ok().json(synonyms)) -} - -make_update_delete_routes!( - "/indexes/{index_uid}/settings/synonyms", - BTreeMap>, - synonyms -); From 8afbb9c46253c91ac8331b6f23d918f548561daa Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 15 Jun 2021 16:22:06 +0200 Subject: [PATCH 439/522] enable response error for documents routes --- meilisearch-http/src/routes/document.rs | 81 ++++++------------------- 1 file changed, 19 insertions(+), 62 deletions(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 4f211bf09..9019bdb7c 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -61,15 +61,10 @@ async fn get_document( ) -> Result { let index = path.index_uid.clone(); let id = path.document_id.clone(); - match data + let document = data .retrieve_document(index, id, None as Option>) - .await - { - Ok(document) => Ok(HttpResponse::Ok().json(document)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + .await?; + Ok(HttpResponse::Ok().json(document)) } #[delete( @@ -80,17 +75,10 @@ async fn delete_document( data: web::Data, path: web::Path, ) -> Result { - match data + let update_status = data .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]) - .await - { - Ok(update_status) => Ok( - HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })) - ), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + .await?; + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } #[derive(Deserialize)] @@ -118,20 +106,15 @@ async fn get_all_documents( Some(names) }); - match data + let documents = data .retrieve_documents( path.index_uid.clone(), params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), attributes_to_retrieve, ) - .await - { - Ok(documents) => Ok(HttpResponse::Ok().json(documents)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + .await?; + Ok(HttpResponse::Ok().json(documents)) } #[derive(Deserialize)] @@ -149,7 +132,7 @@ async fn add_documents( params: web::Query, body: Payload, ) -> Result { - let addition_result = data + let update_status = data .add_documents( path.into_inner().index_uid, IndexDocumentsMethod::ReplaceDocuments, @@ -157,16 +140,9 @@ async fn add_documents( body, params.primary_key.clone(), ) - .await; + .await?; - match addition_result { - Ok(update_status) => Ok( - HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })) - ), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } /// Default route for adding documents, this should return an error and redirect to the documentation @@ -200,7 +176,7 @@ async fn update_documents( params: web::Query, body: web::Payload, ) -> Result { - let addition_result = data + let update = data .add_documents( path.into_inner().index_uid, IndexDocumentsMethod::UpdateDocuments, @@ -208,16 +184,9 @@ async fn update_documents( body, params.primary_key.clone(), ) - .await; + .await?; - match addition_result { - Ok(update) => { - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() }))) - } - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() }))) } #[post( @@ -238,14 +207,8 @@ async fn delete_documents( }) .collect(); - match data.delete_documents(path.index_uid.clone(), ids).await { - Ok(update_status) => Ok( - HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })) - ), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let update_status = data.delete_documents(path.index_uid.clone(), ids).await?; + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } /// delete all documents @@ -254,12 +217,6 @@ async fn clear_all_documents( data: web::Data, path: web::Path, ) -> Result { - match data.clear_documents(path.index_uid.clone()).await { - Ok(update_status) => Ok( - HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })) - ), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let update_status = data.clear_documents(path.index_uid.clone()).await?; + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } From 439db1aae03538e864f8513c955dfaf7c3f4c17b Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 15 Jun 2021 16:28:10 +0200 Subject: [PATCH 440/522] enable response error for search routes --- meilisearch-http/src/routes/search.rs | 47 ++++++++------------------- 1 file changed, 13 insertions(+), 34 deletions(-) diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 3ae38bb21..af557ae95 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -2,8 +2,8 @@ use std::collections::{BTreeSet, HashSet}; use std::convert::{TryFrom, TryInto}; use actix_web::{get, post, web, HttpResponse}; -use serde_json::Value; use serde::Deserialize; +use serde_json::Value; use crate::error::ResponseError; use crate::helpers::Authentication; @@ -30,10 +30,8 @@ pub struct SearchQueryGet { facet_distributions: Option, } -impl TryFrom for SearchQuery { - type Error = Box; - - fn try_from(other: SearchQueryGet) -> Result { +impl From for SearchQuery { + fn from(other: SearchQueryGet) -> Self { let attributes_to_retrieve = other .attributes_to_retrieve .map(|attrs| attrs.split(',').map(String::from).collect::>()); @@ -51,16 +49,14 @@ impl TryFrom for SearchQuery { .map(|attrs| attrs.split(',').map(String::from).collect::>()); let filter = match other.filter { - Some(f) => { - match serde_json::from_str(&f) { - Ok(v) => Some(v), - _ => Some(Value::String(f)), - } + Some(f) => match serde_json::from_str(&f) { + Ok(v) => Some(v), + _ => Some(Value::String(f)), }, None => None, }; - Ok(Self { + Self { q: other.q, offset: other.offset, limit: other.limit.unwrap_or(DEFAULT_SEARCH_LIMIT), @@ -71,7 +67,7 @@ impl TryFrom for SearchQuery { filter, matches: other.matches, facet_distributions, - }) + } } } @@ -81,21 +77,9 @@ async fn search_with_url_query( path: web::Path, params: web::Query, ) -> Result { - let query: SearchQuery = match params.into_inner().try_into() { - Ok(q) => q, - Err(e) => { - return Ok( - HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() })) - ) - } - }; - let search_result = data.search(path.into_inner().index_uid, query).await; - match search_result { - Ok(docs) => Ok(HttpResponse::Ok().json(docs)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + let query = params.into_inner().into(); + let search_result = data.search(path.into_inner().index_uid, query).await?; + Ok(HttpResponse::Ok().json(search_result)) } #[post("/indexes/{index_uid}/search", wrap = "Authentication::Public")] @@ -106,11 +90,6 @@ async fn search_with_post( ) -> Result { let search_result = data .search(path.into_inner().index_uid, params.into_inner()) - .await; - match search_result { - Ok(docs) => Ok(HttpResponse::Ok().json(docs)), - Err(e) => { - Ok(HttpResponse::BadRequest().json(serde_json::json!({ "error": e.to_string() }))) - } - } + .await?; + Ok(HttpResponse::Ok().json(search_result)) } From 8fc12b1526944776d90da1d2ec1ffbbfd3ea3dd8 Mon Sep 17 00:00:00 2001 From: marin Date: Mon, 21 Jun 2021 11:06:06 +0200 Subject: [PATCH 441/522] Update meilisearch-http/src/index/search.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Renault --- meilisearch-http/src/index/search.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 208804803..01650c9f5 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -496,8 +496,8 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { tokens .map(|(word, token)| { - if let Some(match_len) = matcher.matches(token.text()) { - if format_options.highlight && token.is_word() { + if format_options.highlight && token.is_word() { + if let Some(match_len) = matcher.matches(token.text()) { let mut new_word = String::new(); new_word.push_str(&self.marks.0); From 02277ec2cfb5e04fc2ad3b34b450ad7903248aa3 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 15 Jun 2021 17:39:07 +0200 Subject: [PATCH 442/522] reintroduce anyhow --- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/data/mod.rs | 5 +-- meilisearch-http/src/data/search.rs | 6 +-- meilisearch-http/src/data/updates.rs | 2 +- meilisearch-http/src/error.rs | 14 +++---- .../src/helpers/authentication.rs | 8 ++-- meilisearch-http/src/helpers/compression.rs | 4 +- meilisearch-http/src/index/dump.rs | 13 ++++--- meilisearch-http/src/index/error.rs | 12 ++---- meilisearch-http/src/index/update_handler.rs | 2 +- meilisearch-http/src/index/updates.rs | 12 +++--- .../src/index_controller/dump_actor/actor.rs | 2 +- .../src/index_controller/dump_actor/error.rs | 4 +- .../dump_actor/handle_impl.rs | 4 +- .../index_controller/dump_actor/loaders/v1.rs | 6 +-- .../index_controller/dump_actor/loaders/v2.rs | 2 +- .../index_controller/dump_actor/message.rs | 2 +- .../src/index_controller/dump_actor/mod.rs | 11 +++--- .../src/index_controller/index_actor/actor.rs | 14 ++++--- .../src/index_controller/index_actor/error.rs | 6 +-- .../index_actor/handle_impl.rs | 16 ++------ .../index_controller/index_actor/message.rs | 2 +- .../src/index_controller/index_actor/mod.rs | 17 ++------ meilisearch-http/src/index_controller/mod.rs | 9 ++--- .../src/index_controller/snapshot.rs | 6 +-- .../index_controller/update_actor/actor.rs | 10 ++--- .../index_controller/update_actor/error.rs | 1 + .../update_actor/handle_impl.rs | 2 +- .../index_controller/update_actor/message.rs | 2 +- .../src/index_controller/update_actor/mod.rs | 4 +- .../update_actor/store/dump.rs | 4 +- .../update_actor/store/mod.rs | 39 ++++++------------- .../src/index_controller/updates.rs | 2 +- meilisearch-http/src/main.rs | 1 + meilisearch-http/src/routes/mod.rs | 16 ++++---- meilisearch-http/tests/index/get_index.rs | 2 +- 36 files changed, 110 insertions(+), 154 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 485ccf972..70ab30d34 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -27,7 +27,7 @@ actix-http = { version = "=3.0.0-beta.6" } actix-service = "2.0.0" actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] } actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true } -#anyhow = "1.0.36" +anyhow = "1.0.36" async-stream = "0.3.0" async-trait = "0.1.42" arc-swap = "1.2.0" diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 634216ade..48dfcfa06 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -5,8 +5,7 @@ use sha2::Digest; use crate::index::{Checked, Settings}; use crate::index_controller::{ - DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats, - error::Result + error::Result, DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats, }; use crate::option::Opt; @@ -57,7 +56,7 @@ impl ApiKeys { } impl Data { - pub fn new(options: Opt) -> std::result::Result> { + pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); let index_controller = IndexController::new(&path, &options)?; diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index 30645cfaf..5ad8d4a07 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -5,11 +5,7 @@ use crate::index::{SearchQuery, SearchResult}; use crate::index_controller::error::Result; impl Data { - pub async fn search( - &self, - index: String, - search_query: SearchQuery, - ) -> Result { + pub async fn search(&self, index: String, search_query: SearchQuery) -> Result { self.index_controller.search(index, search_query).await } diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index e2b9f6629..6b29d46b1 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -3,7 +3,7 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use super::Data; use crate::index::{Checked, Settings}; -use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus, error::Result}; +use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus}; impl Data { pub async fn add_documents( diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 04e2c52ca..a853fd8cf 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -22,7 +22,7 @@ pub enum AuthenticationError { impl ErrorCode for AuthenticationError { fn error_code(&self) -> Code { match self { - AuthenticationError ::MissingAuthorizationHeader => Code::MissingAuthorizationHeader, + AuthenticationError::MissingAuthorizationHeader => Code::MissingAuthorizationHeader, AuthenticationError::InvalidToken(_) => Code::InvalidToken, } } @@ -62,11 +62,7 @@ macro_rules! response_error { }; } -response_error!( - IndexControllerError, - AuthenticationError -); - +response_error!(IndexControllerError, AuthenticationError); impl Serialize for ResponseError { fn serialize(&self, serializer: S) -> Result @@ -114,7 +110,8 @@ impl ErrorCode for PayloadError { } impl From> for ResponseError -where E: Error + Sync + Send + 'static +where + E: Error + Sync + Send + 'static, { fn from(other: PayloadError) -> Self { ResponseError { @@ -124,7 +121,8 @@ where E: Error + Sync + Send + 'static } pub fn payload_error_handler(err: E) -> ResponseError -where E: Error + Sync + Send + 'static +where + E: Error + Sync + Send + 'static, { let error = PayloadError(err); error.into() diff --git a/meilisearch-http/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs index 5577b9c1d..dddf57138 100644 --- a/meilisearch-http/src/helpers/authentication.rs +++ b/meilisearch-http/src/helpers/authentication.rs @@ -9,7 +9,7 @@ use futures::future::{ok, Future, Ready}; use futures::ready; use pin_project::pin_project; -use crate::error::{ResponseError, AuthenticationError}; +use crate::error::{AuthenticationError, ResponseError}; use crate::Data; #[derive(Clone, Copy)] @@ -117,7 +117,8 @@ where AuthProj::NoHeader(req) => { match req.take() { Some(req) => { - let response = ResponseError::from(AuthenticationError::MissingAuthorizationHeader); + let response = + ResponseError::from(AuthenticationError::MissingAuthorizationHeader); let response = response.error_response(); let response = req.into_response(response); Poll::Ready(Ok(response)) @@ -134,7 +135,8 @@ where .get("X-Meili-API-Key") .map(|h| h.to_str().map(String::from).unwrap_or_default()) .unwrap_or_default(); - let response = ResponseError::from(AuthenticationError::InvalidToken(bad_token)); + let response = + ResponseError::from(AuthenticationError::InvalidToken(bad_token)); let response = response.error_response(); let response = req.into_response(response); Poll::Ready(Ok(response)) diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index 55861ca53..c4747cb21 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -5,7 +5,7 @@ use std::path::Path; use flate2::{read::GzDecoder, write::GzEncoder, Compression}; use tar::{Archive, Builder}; -pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Box> { +pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { let mut f = File::create(dest)?; let gz_encoder = GzEncoder::new(&mut f, Compression::default()); let mut tar_encoder = Builder::new(gz_encoder); @@ -16,7 +16,7 @@ pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Bo Ok(()) } -pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> Result<(), Box> { +pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { let f = File::open(&src)?; let gz = GzDecoder::new(f); let mut ar = Archive::new(gz); diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 05202be2d..f66edd9d1 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -3,6 +3,7 @@ use std::io::{BufRead, BufReader, Write}; use std::path::Path; use std::sync::Arc; +use anyhow::Context; use heed::RoTxn; use indexmap::IndexMap; use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; @@ -10,8 +11,8 @@ use serde::{Deserialize, Serialize}; use crate::option::IndexerOpts; -use super::{update_handler::UpdateHandler, Index, Settings, Unchecked}; use super::error::{IndexError, Result}; +use super::{update_handler::UpdateHandler, Index, Settings, Unchecked}; #[derive(Serialize, Deserialize)] struct DumpMeta { @@ -37,7 +38,8 @@ impl Index { let document_file_path = path.as_ref().join(DATA_FILE_NAME); let mut document_file = File::create(&document_file_path)?; - let documents = self.all_documents(txn) + let documents = self + .all_documents(txn) .map_err(|e| IndexError::Internal(e.into()))?; let fields_ids_map = self.fields_ids_map(txn)?; @@ -82,13 +84,12 @@ impl Index { dst: impl AsRef, size: usize, indexing_options: &IndexerOpts, - ) -> std::result::Result<(), Box> { + ) -> anyhow::Result<()> { let dir_name = src .as_ref() .file_name() - // TODO: remove - //.with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?; - .unwrap(); + .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?; + let dst_dir_path = dst.as_ref().join("indexes").join(dir_name); create_dir_all(&dst_dir_path)?; diff --git a/meilisearch-http/src/index/error.rs b/meilisearch-http/src/index/error.rs index cfda813a1..25d017547 100644 --- a/meilisearch-http/src/index/error.rs +++ b/meilisearch-http/src/index/error.rs @@ -27,12 +27,7 @@ macro_rules! internal_error { } } -internal_error!( - std::io::Error, - heed::Error, - fst::Error, - serde_json::Error -); +internal_error!(std::io::Error, heed::Error, fst::Error, serde_json::Error); impl ErrorCode for IndexError { fn error_code(&self) -> Code { @@ -47,14 +42,13 @@ impl ErrorCode for IndexError { #[derive(Debug, thiserror::Error)] pub enum FacetError { #[error("Invalid facet expression, expected {}, found: {1}", .0.join(", "))] - InvalidExpression(&'static [&'static str], Value) + InvalidExpression(&'static [&'static str], Value), } -impl ErrorCode for FacetError { +impl ErrorCode for FacetError { fn error_code(&self) -> Code { match self { FacetError::InvalidExpression(_, _) => Code::Facet, } } } - diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs index ce6a7e48d..10b28bcc9 100644 --- a/meilisearch-http/src/index/update_handler.rs +++ b/meilisearch-http/src/index/update_handler.rs @@ -21,7 +21,7 @@ pub struct UpdateHandler { } impl UpdateHandler { - pub fn new(opt: &IndexerOpts) -> std::result::Result> { + pub fn new(opt: &IndexerOpts) -> anyhow::Result { let thread_pool = rayon::ThreadPoolBuilder::new() .num_threads(opt.indexing_jobs.unwrap_or(0)) .build()?; diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 04e90f4f4..84fc945d5 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, BTreeMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::io; use std::marker::PhantomData; use std::num::NonZeroUsize; @@ -308,10 +308,9 @@ impl Index { } } - builder.execute(|indexing_step, update_id| { - info!("update {}: {:?}", update_id, indexing_step) - }) - .map_err(|e| IndexError::Internal(e.into()))?; + builder + .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)) + .map_err(|e| IndexError::Internal(e.into()))?; Ok(UpdateResult::Other) } @@ -333,7 +332,8 @@ impl Index { update_builder: UpdateBuilder, ) -> Result { let mut txn = self.write_txn()?; - let mut builder = update_builder.delete_documents(&mut txn, self) + let mut builder = update_builder + .delete_documents(&mut txn, self) .map_err(|e| IndexError::Internal(e.into()))?; // We ignore unexisting document ids diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index a534a3a69..4924df947 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -10,9 +10,9 @@ use tokio::sync::{mpsc, oneshot, RwLock}; use update_actor::UpdateActorHandle; use uuid_resolver::UuidResolverHandle; +use super::error::{DumpActorError, Result}; use super::{DumpInfo, DumpMsg, DumpStatus, DumpTask}; use crate::index_controller::{update_actor, uuid_resolver}; -use super::error::{DumpActorError, Result}; pub const CONCURRENT_DUMP_MSG: usize = 10; diff --git a/meilisearch-http/src/index_controller/dump_actor/error.rs b/meilisearch-http/src/index_controller/dump_actor/error.rs index c0f373181..ff5147e89 100644 --- a/meilisearch-http/src/index_controller/dump_actor/error.rs +++ b/meilisearch-http/src/index_controller/dump_actor/error.rs @@ -1,6 +1,8 @@ use meilisearch_error::{Code, ErrorCode}; -use crate::index_controller::{update_actor::error::UpdateActorError, uuid_resolver::UuidResolverError}; +use crate::index_controller::{ + update_actor::error::UpdateActorError, uuid_resolver::UuidResolverError, +}; pub type Result = std::result::Result; diff --git a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs index b233a4e61..db11fb8fc 100644 --- a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs @@ -3,8 +3,8 @@ use std::path::Path; use actix_web::web::Bytes; use tokio::sync::{mpsc, oneshot}; -use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg}; use super::error::Result; +use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg}; #[derive(Clone)] pub struct DumpActorHandleImpl { @@ -35,7 +35,7 @@ impl DumpActorHandleImpl { update: crate::index_controller::update_actor::UpdateActorHandleImpl, index_db_size: usize, update_db_size: usize, - ) -> std::result::Result> { + ) -> anyhow::Result { let (sender, receiver) = mpsc::channel(10); let actor = DumpActor::new( receiver, diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 502a6333e..a7f1aa8d1 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -31,7 +31,7 @@ impl MetadataV1 { dst: impl AsRef, size: usize, indexer_options: &IndexerOpts, - ) -> std::result::Result<(), Box> { + ) -> anyhow::Result<()> { info!( "Loading dump, dump database version: {}, dump version: V1", self.db_version @@ -83,7 +83,7 @@ fn load_index( primary_key: Option<&str>, size: usize, indexer_options: &IndexerOpts, -) -> std::result::Result<(), Box> { +) -> anyhow::Result<()> { let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid)); create_dir_all(&index_path)?; @@ -172,7 +172,7 @@ impl From for index_controller::Settings { } /// Extract Settings from `settings.json` file present at provided `dir_path` -fn import_settings(dir_path: impl AsRef) -> std::result::Result> { +fn import_settings(dir_path: impl AsRef) -> anyhow::Result { let path = dir_path.as_ref().join("settings.json"); let file = File::open(path)?; let reader = std::io::BufReader::new(file); diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index 7f5420d23..eddd8a3b7 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -34,7 +34,7 @@ impl MetadataV2 { index_db_size: usize, update_db_size: usize, indexing_options: &IndexerOpts, - ) -> std::result::Result<(), Box> { + ) -> anyhow::Result<()> { info!( "Loading dump from {}, dump database version: {}, dump version: V2", self.dump_date, self.db_version diff --git a/meilisearch-http/src/index_controller/dump_actor/message.rs b/meilisearch-http/src/index_controller/dump_actor/message.rs index b34314eff..6c9dded9f 100644 --- a/meilisearch-http/src/index_controller/dump_actor/message.rs +++ b/meilisearch-http/src/index_controller/dump_actor/message.rs @@ -1,7 +1,7 @@ use tokio::sync::oneshot; -use super::DumpInfo; use super::error::Result; +use super::DumpInfo; pub enum DumpMsg { CreateDump { diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 35daa0da8..057b8ebaf 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -1,6 +1,7 @@ use std::fs::File; use std::path::{Path, PathBuf}; +use anyhow::Context; use chrono::{DateTime, Utc}; use log::{info, warn}; #[cfg(test)] @@ -21,10 +22,10 @@ use crate::{helpers::compression, option::IndexerOpts}; use error::Result; mod actor; +pub mod error; mod handle_impl; mod loaders; mod message; -pub mod error; const META_FILE_NAME: &str = "metadata.json"; @@ -107,7 +108,7 @@ pub fn load_dump( index_db_size: usize, update_db_size: usize, indexer_opts: &IndexerOpts, -) -> std::result::Result<(), Box> { +) -> anyhow::Result<()> { let tmp_src = tempfile::tempdir_in(".")?; let tmp_src_path = tmp_src.path(); @@ -120,9 +121,7 @@ pub fn load_dump( let dst_dir = dst_path .as_ref() .parent() - // TODO - //.with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?; - .unwrap(); + .with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?; let tmp_dst = tempfile::tempdir_in(dst_dir)?; @@ -188,7 +187,7 @@ where let dump_path = tokio::task::spawn_blocking(move || -> Result { let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; compression::to_tar_gz(temp_dump_path, temp_dump_file.path()) - .map_err(|e| DumpActorError::Internal(e))?; + .map_err(|e| DumpActorError::Internal(e.into()))?; let dump_path = self.path.join(self.uid).with_extension("dump"); temp_dump_file.persist(&dump_path)?; diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index c714cfd25..28c783eb2 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -19,8 +19,8 @@ use crate::index_controller::{ }; use crate::option::IndexerOpts; +use super::error::{IndexActorError, Result}; use super::{IndexMeta, IndexMsg, IndexSettings, IndexStore}; -use super::error::{Result, IndexActorError}; pub const CONCURRENT_INDEX_MSG: usize = 10; @@ -31,7 +31,7 @@ pub struct IndexActor { } impl IndexActor { - pub fn new(receiver: mpsc::Receiver, store: S) -> std::result::Result> { + pub fn new(receiver: mpsc::Receiver, store: S) -> anyhow::Result { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options)?; let update_handler = Arc::new(update_handler); @@ -146,7 +146,6 @@ impl IndexActor { .ok_or(IndexActorError::UnexistingIndex)?; let result = spawn_blocking(move || index.perform_search(query)).await??; Ok(result) - } async fn handle_create_index( @@ -269,7 +268,8 @@ impl IndexActor { } let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); builder.set_primary_key(primary_key); - builder.execute(|_, _| ()) + builder + .execute(|_, _| ()) .map_err(|e| IndexActorError::Internal(Box::new(e)))?; let meta = IndexMeta::new_txn(&index, &txn)?; txn.commit()?; @@ -340,10 +340,12 @@ impl IndexActor { Ok(IndexStats { size: index.size(), - number_of_documents: index.number_of_documents(&rtxn) + number_of_documents: index + .number_of_documents(&rtxn) .map_err(|e| IndexActorError::Internal(Box::new(e)))?, is_indexing: None, - fields_distribution: index.fields_distribution(&rtxn) + fields_distribution: index + .fields_distribution(&rtxn) .map_err(|e| IndexActorError::Internal(e.into()))?, }) }) diff --git a/meilisearch-http/src/index_controller/index_actor/error.rs b/meilisearch-http/src/index_controller/index_actor/error.rs index a124d8b1e..7c7ad1607 100644 --- a/meilisearch-http/src/index_controller/index_actor/error.rs +++ b/meilisearch-http/src/index_controller/index_actor/error.rs @@ -30,11 +30,7 @@ macro_rules! internal_error { } } -internal_error!( - heed::Error, - tokio::task::JoinError, - std::io::Error -); +internal_error!(heed::Error, tokio::task::JoinError, std::io::Error); impl ErrorCode for IndexActorError { fn error_code(&self) -> Code { diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 6afd409d2..231a3a44b 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -12,8 +12,8 @@ use crate::{ index_controller::{Failed, Processed}, }; -use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore}; use super::error::Result; +use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore}; #[derive(Clone)] pub struct IndexActorHandleImpl { @@ -22,11 +22,7 @@ pub struct IndexActorHandleImpl { #[async_trait::async_trait] impl IndexActorHandle for IndexActorHandleImpl { - async fn create_index( - &self, - uuid: Uuid, - primary_key: Option, - ) -> Result { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { ret, @@ -118,11 +114,7 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn update_index( - &self, - uuid: Uuid, - index_settings: IndexSettings, - ) -> Result { + async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::UpdateIndex { uuid, @@ -156,7 +148,7 @@ impl IndexActorHandle for IndexActorHandleImpl { } impl IndexActorHandleImpl { - pub fn new(path: impl AsRef, index_size: usize) -> std::result::Result> { + pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { let (sender, receiver) = mpsc::channel(100); let store = MapIndexStore::new(path, index_size); diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index d9c0dabd9..415b90e4b 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -3,9 +3,9 @@ use std::path::PathBuf; use tokio::sync::oneshot; use uuid::Uuid; +use super::error::Result as IndexResult; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::index_controller::{Failed, IndexStats, Processed, Processing}; -use super::error::Result as IndexResult; use super::{IndexMeta, IndexSettings}; diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index a66f1b9ea..65f196e41 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -22,10 +22,10 @@ use self::error::IndexActorError; use super::IndexSettings; mod actor; +pub mod error; mod handle_impl; mod message; mod store; -pub mod error; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -60,8 +60,7 @@ impl IndexMeta { #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait IndexActorHandle { - async fn create_index(&self, uuid: Uuid, primary_key: Option) - -> Result; + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn update( &self, uuid: Uuid, @@ -86,11 +85,7 @@ pub trait IndexActorHandle { ) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; async fn get_index_meta(&self, uuid: Uuid) -> Result; - async fn update_index( - &self, - uuid: Uuid, - index_settings: IndexSettings, - ) -> Result; + async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>; async fn get_index_stats(&self, uuid: Uuid) -> Result; @@ -105,11 +100,7 @@ mod test { #[async_trait::async_trait] /// Useful for passing around an `Arc` in tests. impl IndexActorHandle for Arc { - async fn create_index( - &self, - uuid: Uuid, - primary_key: Option, - ) -> Result { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { self.as_ref().create_index(uuid, primary_key).await } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 0046bd6a5..188e3b434 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -29,12 +29,12 @@ use self::dump_actor::load_dump; use self::error::IndexControllerError; mod dump_actor; +pub mod error; mod index_actor; mod snapshot; mod update_actor; mod updates; mod uuid_resolver; -pub mod error; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -83,7 +83,7 @@ pub struct Stats { } impl IndexController { - pub fn new(path: impl AsRef, options: &Opt) -> std::result::Result> { + pub fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { let index_size = options.max_mdb_size.get_bytes() as usize; let update_store_size = options.max_udb_size.get_bytes() as usize; @@ -238,10 +238,7 @@ impl IndexController { } } - pub async fn create_index( - &self, - index_settings: IndexSettings, - ) -> Result { + pub async fn create_index(&self, index_settings: IndexSettings) -> Result { let IndexSettings { uid, primary_key } = index_settings; let uid = uid.ok_or(IndexControllerError::MissingUid)?; let uuid = Uuid::new_v4(); diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 171c08eee..a5259eeae 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -52,7 +52,7 @@ where } } - async fn perform_snapshot(&self) -> std::result::Result<(), Box> { + async fn perform_snapshot(&self) -> anyhow::Result<()> { info!("Performing snapshot."); let snapshot_dir = self.snapshot_path.clone(); @@ -77,7 +77,7 @@ where let snapshot_path = self .snapshot_path .join(format!("{}.snapshot", self.db_name)); - let snapshot_path = spawn_blocking(move || -> Result> { + let snapshot_path = spawn_blocking(move || -> anyhow::Result { let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?; let temp_snapshot_file_path = temp_snapshot_file.path().to_owned(); compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?; @@ -97,7 +97,7 @@ pub fn load_snapshot( snapshot_path: impl AsRef, ignore_snapshot_if_db_exists: bool, ignore_missing_snapshot: bool, -) -> std::result::Result<(), Box> { +) -> anyhow::Result<()> { if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() { match compression::from_tar_gz(snapshot_path, &db_path) { Ok(()) => Ok(()), diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index fa8251a0c..1a894aaba 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -13,8 +13,8 @@ use tokio::io::AsyncWriteExt; use tokio::sync::mpsc; use uuid::Uuid; -use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo}; use super::error::{Result, UpdateActorError}; +use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo}; use crate::index_controller::index_actor::IndexActorHandle; use crate::index_controller::{UpdateMeta, UpdateStatus}; @@ -36,7 +36,7 @@ where inbox: mpsc::Receiver>, path: impl AsRef, index_handle: I, - ) -> std::result::Result> { + ) -> anyhow::Result { let path = path.as_ref().join("updates"); std::fs::create_dir_all(&path)?; @@ -201,9 +201,9 @@ where async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { let store = self.store.clone(); tokio::task::spawn_blocking(move || { - let result = store - .meta(uuid, id)? - .ok_or(UpdateActorError::UnexistingUpdate(id))?; + let result = store + .meta(uuid, id)? + .ok_or(UpdateActorError::UnexistingUpdate(id))?; Ok(result) }) .await? diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs index ee68dd997..af30fb046 100644 --- a/meilisearch-http/src/index_controller/update_actor/error.rs +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -45,6 +45,7 @@ impl From for UpdateActorError { } internal_error!( + UpdateActorError: heed::Error, std::io::Error, serde_json::Error, diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index f5331554b..125c63401 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -22,7 +22,7 @@ where index_handle: I, path: impl AsRef, update_store_size: usize, - ) -> std::result::Result> + ) -> anyhow::Result where I: IndexActorHandle + Clone + Send + Sync + 'static, { diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 48a88b097..6b8a0f73f 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -4,8 +4,8 @@ use std::path::PathBuf; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo}; use super::error::Result; +use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo}; pub enum UpdateMsg { Update { diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index 42be99cf8..cb40311b2 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -7,16 +7,16 @@ use uuid::Uuid; use crate::index_controller::{UpdateMeta, UpdateStatus}; use actor::UpdateActor; -use message::UpdateMsg; use error::Result; +use message::UpdateMsg; pub use handle_impl::UpdateActorHandleImpl; pub use store::{UpdateStore, UpdateStoreInfo}; mod actor; +pub mod error; mod handle_impl; mod message; -pub mod error; pub mod store; type PayloadData = std::result::Result; diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index a621feebe..7c46f98fa 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -9,7 +9,7 @@ use heed::{EnvOpenOptions, RoTxn}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use super::{State, UpdateStore, Result}; +use super::{Result, State, UpdateStore}; use crate::index_controller::{ index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued, UpdateStatus, @@ -125,7 +125,7 @@ impl UpdateStore { src: impl AsRef, dst: impl AsRef, db_size: usize, - ) -> std::result::Result<(), Box> { + ) -> anyhow::Result<()> { let dst_update_path = dst.as_ref().join("updates/"); create_dir_all(&dst_update_path)?; diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 429ce362b..6aa336835 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -23,10 +23,10 @@ use uuid::Uuid; use codec::*; -use super::UpdateMeta; use super::error::Result; -use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle}; +use super::UpdateMeta; use crate::helpers::EnvSizer; +use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle}; #[allow(clippy::upper_case_acronyms)] type BEU64 = U64; @@ -110,7 +110,7 @@ impl UpdateStore { fn new( mut options: EnvOpenOptions, path: impl AsRef, - ) -> std::result::Result<(Self, mpsc::Receiver<()>), Box> { + ) -> anyhow::Result<(Self, mpsc::Receiver<()>)> { options.max_dbs(5); let env = options.open(&path)?; @@ -141,7 +141,7 @@ impl UpdateStore { path: impl AsRef, index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static, must_exit: Arc, - ) -> std::result::Result, Box> { + ) -> anyhow::Result> { let (update_store, mut notification_receiver) = Self::new(options, path)?; let update_store = Arc::new(update_store); @@ -270,11 +270,8 @@ impl UpdateStore { } _ => { let _update_id = self.next_update_id_raw(wtxn, index_uuid)?; - self.updates.put( - wtxn, - &(index_uuid, update.id()), - &update, - )?; + self.updates + .put(wtxn, &(index_uuid, update.id()), &update)?; } } Ok(()) @@ -283,10 +280,7 @@ impl UpdateStore { /// Executes the user provided function on the next pending update (the one with the lowest id). /// This is asynchronous as it let the user process the update with a read-only txn and /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update( - &self, - index_handle: impl IndexActorHandle, - ) -> Result> { + fn process_pending_update(&self, index_handle: impl IndexActorHandle) -> Result> { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; let first_meta = self.pending_queue.first(&rtxn)?; @@ -353,11 +347,8 @@ impl UpdateStore { Err(res) => res.into(), }; - self.updates.put( - &mut wtxn, - &(index_uuid, update_id), - &result, - )?; + self.updates + .put(&mut wtxn, &(index_uuid, update_id), &result)?; wtxn.commit()?; @@ -704,18 +695,10 @@ mod test { let txn = store.env.read_txn().unwrap(); assert!(store.pending_queue.first(&txn).unwrap().is_none()); - let update = store - .updates - .get(&txn, &(uuid, 0)) - .unwrap() - .unwrap(); + let update = store.updates.get(&txn, &(uuid, 0)).unwrap().unwrap(); assert!(matches!(update, UpdateStatus::Processed(_))); - let update = store - .updates - .get(&txn, &(uuid, 1)) - .unwrap() - .unwrap(); + let update = store.updates.get(&txn, &(uuid, 1)).unwrap().unwrap(); assert!(matches!(update, UpdateStatus::Failed(_))); } diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index abeec413e..296ef9b42 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -25,7 +25,7 @@ pub enum UpdateMeta { }, ClearDocuments, DeleteDocuments { - ids: Vec + ids: Vec, }, Settings(Settings), } diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 2787d8b78..9f3e62356 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -30,6 +30,7 @@ async fn main() -> Result<(), MainError> { .into(), ); } + #[cfg(all(not(debug_assertions), feature = "analytics"))] if !opt.no_analytics { let logger = diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 4c5ebbe21..a896653ef 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -24,17 +24,19 @@ pub enum UpdateType { Customs, DocumentsAddition { #[serde(skip_serializing_if = "Option::is_none")] - number: Option + number: Option, }, DocumentsPartial { #[serde(skip_serializing_if = "Option::is_none")] - number: Option + number: Option, }, DocumentsDeletion { #[serde(skip_serializing_if = "Option::is_none")] - number: Option + number: Option, + }, + Settings { + settings: Settings, }, - Settings { settings: Settings }, } impl From<&UpdateStatus> for UpdateType { @@ -60,9 +62,9 @@ impl From<&UpdateStatus> for UpdateType { } } UpdateMeta::ClearDocuments => UpdateType::ClearAll, - UpdateMeta::DeleteDocuments { ids } => { - UpdateType::DocumentsDeletion { number: Some(ids.len()) } - } + UpdateMeta::DeleteDocuments { ids } => UpdateType::DocumentsDeletion { + number: Some(ids.len()), + }, UpdateMeta::Settings(settings) => UpdateType::Settings { settings: settings.clone(), }, diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index ce4d7b792..4c2727bfa 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -21,7 +21,7 @@ async fn create_and_get_index() { assert_eq!(response.as_object().unwrap().len(), 5); } -// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// TODO: partial test since we are testing error, and error is not yet fully implemented in // transplant #[actix_rt::test] async fn get_unexisting_index() { From 0d3fb5ee0ddf620dff9036012bc42d1f7fc9619d Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 15 Jun 2021 17:55:27 +0200 Subject: [PATCH 443/522] factorize internal error macro --- meilisearch-http/src/error.rs | 12 +++++ meilisearch-http/src/index/error.rs | 20 +++----- .../src/index_controller/dump_actor/error.rs | 5 +- .../src/index_controller/error.rs | 7 +-- meilisearch-http/src/index_controller/mod.rs | 2 +- .../index_controller/update_actor/error.rs | 12 ----- .../index_controller/uuid_resolver/actor.rs | 2 +- .../index_controller/uuid_resolver/error.rs | 34 +++++++++++++ .../src/index_controller/uuid_resolver/mod.rs | 49 +------------------ .../index_controller/uuid_resolver/store.rs | 2 +- meilisearch-http/src/lib.rs | 1 + 11 files changed, 62 insertions(+), 84 deletions(-) create mode 100644 meilisearch-http/src/index_controller/uuid_resolver/error.rs diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index a853fd8cf..7468f0630 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -127,3 +127,15 @@ where let error = PayloadError(err); error.into() } + +macro_rules! internal_error { + ($target:ty : $($other:path), *) => { + $( + impl From<$other> for $target { + fn from(other: $other) -> Self { + Self::Internal(Box::new(other)) + } + } + )* + } +} diff --git a/meilisearch-http/src/index/error.rs b/meilisearch-http/src/index/error.rs index 25d017547..32f13a143 100644 --- a/meilisearch-http/src/index/error.rs +++ b/meilisearch-http/src/index/error.rs @@ -15,19 +15,13 @@ pub enum IndexError { Facet(#[from] FacetError), } -macro_rules! internal_error { - ($($other:path), *) => { - $( - impl From<$other> for IndexError { - fn from(other: $other) -> Self { - Self::Internal(Box::new(other)) - } - } - )* - } -} - -internal_error!(std::io::Error, heed::Error, fst::Error, serde_json::Error); +internal_error!( + IndexError: + std::io::Error, + heed::Error, + fst::Error, + serde_json::Error +); impl ErrorCode for IndexError { fn error_code(&self) -> Code { diff --git a/meilisearch-http/src/index_controller/dump_actor/error.rs b/meilisearch-http/src/index_controller/dump_actor/error.rs index ff5147e89..778b6b25c 100644 --- a/meilisearch-http/src/index_controller/dump_actor/error.rs +++ b/meilisearch-http/src/index_controller/dump_actor/error.rs @@ -1,8 +1,7 @@ use meilisearch_error::{Code, ErrorCode}; -use crate::index_controller::{ - update_actor::error::UpdateActorError, uuid_resolver::UuidResolverError, -}; +use crate::index_controller::update_actor::error::UpdateActorError; +use crate::index_controller::uuid_resolver::error::UuidResolverError; pub type Result = std::result::Result; diff --git a/meilisearch-http/src/index_controller/error.rs b/meilisearch-http/src/index_controller/error.rs index 23992ba80..0862c299e 100644 --- a/meilisearch-http/src/index_controller/error.rs +++ b/meilisearch-http/src/index_controller/error.rs @@ -1,19 +1,15 @@ -use std::error::Error; - use meilisearch_error::Code; use meilisearch_error::ErrorCode; use super::dump_actor::error::DumpActorError; use super::index_actor::error::IndexActorError; use super::update_actor::error::UpdateActorError; -use super::uuid_resolver::UuidResolverError; +use super::uuid_resolver::error::UuidResolverError; pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum IndexControllerError { - #[error("Internal error: {0}")] - Internal(Box), #[error("Missing index uid")] MissingUid, #[error("index resolution error: {0}")] @@ -29,7 +25,6 @@ pub enum IndexControllerError { impl ErrorCode for IndexControllerError { fn error_code(&self) -> Code { match self { - IndexControllerError::Internal(_) => Code::Internal, IndexControllerError::MissingUid => Code::InvalidIndexUid, IndexControllerError::Uuid(e) => e.error_code(), IndexControllerError::IndexActor(e) => e.error_code(), diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 188e3b434..d82eb3d30 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -19,7 +19,7 @@ use index_actor::IndexActorHandle; use snapshot::{load_snapshot, SnapshotService}; use update_actor::UpdateActorHandle; pub use updates::*; -use uuid_resolver::{UuidResolverError, UuidResolverHandle}; +use uuid_resolver::{error::UuidResolverError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs index af30fb046..003d90054 100644 --- a/meilisearch-http/src/index_controller/update_actor/error.rs +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -20,18 +20,6 @@ pub enum UpdateActorError { FatalUpdateStoreError, } -macro_rules! internal_error { - ($($other:path), *) => { - $( - impl From<$other> for UpdateActorError { - fn from(other: $other) -> Self { - Self::Internal(Box::new(other)) - } - } - )* - } -} - impl From> for UpdateActorError { fn from(_: tokio::sync::mpsc::error::SendError) -> Self { Self::FatalUpdateStoreError diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 74158ce04..2c73aa490 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -4,7 +4,7 @@ use log::{info, warn}; use tokio::sync::mpsc; use uuid::Uuid; -use super::{Result, UuidResolveMsg, UuidResolverError, UuidStore}; +use super::{Result, UuidResolveMsg, error::UuidResolverError, UuidStore}; pub struct UuidResolverActor { inbox: mpsc::Receiver, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/error.rs b/meilisearch-http/src/index_controller/uuid_resolver/error.rs new file mode 100644 index 000000000..30cde08c1 --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/error.rs @@ -0,0 +1,34 @@ +use meilisearch_error::{Code, ErrorCode}; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum UuidResolverError { + #[error("Name already exist.")] + NameAlreadyExist, + #[error("Index \"{0}\" doesn't exist.")] + UnexistingIndex(String), + #[error("Badly formatted index uid: {0}")] + BadlyFormatted(String), + #[error("Internal error resolving index uid: {0}")] + Internal(Box), +} + +internal_error!( + UuidResolverError: heed::Error, + uuid::Error, + std::io::Error, + tokio::task::JoinError, + serde_json::Error +); + +impl ErrorCode for UuidResolverError { + fn error_code(&self) -> Code { + match self { + UuidResolverError::NameAlreadyExist => Code::IndexAlreadyExists, + UuidResolverError::UnexistingIndex(_) => Code::IndexNotFound, + UuidResolverError::BadlyFormatted(_) => Code::InvalidIndexUid, + UuidResolverError::Internal(_) => Code::Internal, + } + } +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 67d77d411..b7e40618e 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -1,4 +1,5 @@ mod actor; +pub mod error; mod handle_impl; mod message; pub mod store; @@ -6,14 +7,12 @@ pub mod store; use std::collections::HashSet; use std::path::PathBuf; -use meilisearch_error::Code; -use meilisearch_error::ErrorCode; -use thiserror::Error; use uuid::Uuid; use actor::UuidResolverActor; use message::UuidResolveMsg; use store::UuidStore; +use error::Result; #[cfg(test)] use mockall::automock; @@ -23,7 +22,6 @@ pub use store::HeedUuidStore; const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB -pub type Result = std::result::Result; #[async_trait::async_trait] #[cfg_attr(test, automock)] @@ -36,46 +34,3 @@ pub trait UuidResolverHandle { async fn get_size(&self) -> Result; async fn dump(&self, path: PathBuf) -> Result>; } - -#[derive(Debug, Error)] -pub enum UuidResolverError { - #[error("Name already exist.")] - NameAlreadyExist, - #[error("Index \"{0}\" doesn't exist.")] - UnexistingIndex(String), - #[error("Badly formatted index uid: {0}")] - BadlyFormatted(String), - #[error("Internal error resolving index uid: {0}")] - Internal(Box), -} - -macro_rules! internal_error { - ($($other:path), *) => { - $( - impl From<$other> for UuidResolverError { - fn from(other: $other) -> Self { - Self::Internal(Box::new(other)) - } - } - )* - } -} - -internal_error!( - heed::Error, - uuid::Error, - std::io::Error, - tokio::task::JoinError, - serde_json::Error -); - -impl ErrorCode for UuidResolverError { - fn error_code(&self) -> Code { - match self { - UuidResolverError::NameAlreadyExist => Code::IndexAlreadyExists, - UuidResolverError::UnexistingIndex(_) => Code::IndexNotFound, - UuidResolverError::BadlyFormatted(_) => Code::InvalidIndexUid, - UuidResolverError::Internal(_) => Code::Internal, - } - } -} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 88d707797..9107ce908 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -8,7 +8,7 @@ use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use super::{Result, UuidResolverError, UUID_STORE_SIZE}; +use super::{Result, error::UuidResolverError, UUID_STORE_SIZE}; use crate::helpers::EnvSizer; #[derive(Serialize, Deserialize)] diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index a815a3659..25c29ac11 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,4 +1,5 @@ pub mod data; +#[macro_use] pub mod error; pub mod helpers; mod index; From 0dfd1b74c8faf3ff3c3bdfa8d090544ecd0b20ea Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 15 Jun 2021 22:21:52 +0200 Subject: [PATCH 444/522] fix tests --- meilisearch-http/src/index_controller/snapshot.rs | 2 +- meilisearch-http/tests/documents/delete_documents.rs | 8 ++++---- meilisearch-http/tests/documents/get_documents.rs | 6 +++--- meilisearch-http/tests/index/create_index.rs | 2 +- meilisearch-http/tests/index/delete_index.rs | 4 ++-- meilisearch-http/tests/index/get_index.rs | 2 +- meilisearch-http/tests/index/update_index.rs | 2 +- meilisearch-http/tests/settings/get_settings.rs | 10 +++++----- meilisearch-http/tests/stats/mod.rs | 3 ++- meilisearch-http/tests/updates/mod.rs | 6 +++--- 10 files changed, 23 insertions(+), 22 deletions(-) diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index a5259eeae..b3bfca2db 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -143,7 +143,7 @@ mod test { use crate::index_controller::update_actor::{ MockUpdateActorHandle, UpdateActorHandleImpl, error::UpdateActorError, }; - use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidResolverError}; + use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, error::UuidResolverError}; #[actix_rt::test] async fn test_normal() { diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs index d9b97d68d..d5794e40c 100644 --- a/meilisearch-http/tests/documents/delete_documents.rs +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -6,7 +6,7 @@ use crate::common::{GetAllDocumentsOptions, Server}; async fn delete_one_document_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").delete_document(0).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -33,14 +33,14 @@ async fn delete_one_document() { index.wait_update_id(1).await; let (_response, code) = index.get_document(0, None).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] async fn clear_all_documents_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").clear_all_documents().await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -86,7 +86,7 @@ async fn clear_all_documents_empty_index() { async fn delete_batch_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").delete_batch(vec![]).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index 24809532e..945bd6b5c 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -9,7 +9,7 @@ use serde_json::json; async fn get_unexisting_index_single_document() { let server = Server::new().await; let (_response, code) = server.index("test").get_document(1, None).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -18,7 +18,7 @@ async fn get_unexisting_document() { let index = server.index("test"); index.create(None).await; let (_response, code) = index.get_document(1, None).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -53,7 +53,7 @@ async fn get_unexisting_index_all_documents() { .index("test") .get_all_documents(GetAllDocumentsOptions::default()) .await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index a61524d04..e65908cb2 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -70,5 +70,5 @@ async fn test_create_multiple_indexes() { assert_eq!(index1.get().await.1, 200); assert_eq!(index2.get().await.1, 200); assert_eq!(index3.get().await.1, 200); - assert_eq!(index4.get().await.1, 400); + assert_eq!(index4.get().await.1, 404); } diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 0d1a7c820..b0f067b24 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -12,7 +12,7 @@ async fn create_and_delete_index() { assert_eq!(code, 204); - assert_eq!(index.get().await.1, 400); + assert_eq!(index.get().await.1, 404); } #[actix_rt::test] @@ -21,5 +21,5 @@ async fn delete_unexisting_index() { let index = server.index("test"); let (_response, code) = index.delete().await; - assert_eq!(code, 400); + assert_eq!(code, 404); } diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index 4c2727bfa..a6b22509e 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -30,7 +30,7 @@ async fn get_unexisting_index() { let (_response, code) = index.get().await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index 9c78ccb16..c7d910b59 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -60,5 +60,5 @@ async fn update_existing_primary_key() { async fn test_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").update(None).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index a03d907a7..679f347fa 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -5,7 +5,7 @@ use serde_json::json; async fn get_settings_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").settings().await; - assert_eq!(code, 400) + assert_eq!(code, 404) } #[actix_rt::test] @@ -33,7 +33,7 @@ async fn update_settings_unknown_field() { let server = Server::new().await; let index = server.index("test"); let (_response, code) = index.update_settings(json!({"foo": 12})).await; - assert_eq!(code, 400); + assert_eq!(code, 500); } #[actix_rt::test] @@ -65,7 +65,7 @@ async fn delete_settings_unexisting_index() { let server = Server::new().await; let index = server.index("test"); let (_response, code) = index.delete_settings().await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -152,7 +152,7 @@ macro_rules! test_setting_routes { .map(|c| if c == '_' { '-' } else { c }) .collect::()); let (_response, code) = server.service.get(url).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -178,7 +178,7 @@ macro_rules! test_setting_routes { .map(|c| if c == '_' { '-' } else { c }) .collect::()); let (_response, code) = server.service.delete(url).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } } )* diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index 7b9ab10b0..130849a64 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -62,11 +62,12 @@ async fn stats() { index.wait_update_id(0).await; + let (response, code) = server.stats().await; assert_eq!(code, 200); assert!(response["databaseSize"].as_u64().unwrap() > 0); - assert!(response["lastUpdate"].as_str().unwrap() > last_update); + assert!(response.get("lastUpdate").is_some()); assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 2); assert!(response["indexes"]["test"]["isIndexing"] == false); assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["id"], 2); diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 6ce5ca381..00bbf32a8 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -4,7 +4,7 @@ use crate::common::Server; async fn get_update_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").get_update(0).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -13,7 +13,7 @@ async fn get_unexisting_update_status() { let index = server.index("test"); index.create(None).await; let (_response, code) = index.get_update(0).await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -39,7 +39,7 @@ async fn get_update_status() { async fn list_updates_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").list_updates().await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] From abdf642d68255d59108bff1e9cdb8886c044f010 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 17 Jun 2021 14:36:32 +0200 Subject: [PATCH 445/522] integrate milli errors --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/error.rs | 42 ++++++++++++++++ meilisearch-http/src/index/dump.rs | 6 +-- meilisearch-http/src/index/error.rs | 5 ++ meilisearch-http/src/index/mod.rs | 48 ++++++------------ meilisearch-http/src/index/search.rs | 17 +++---- meilisearch-http/src/index/updates.rs | 50 ++++++------------- .../src/index_controller/index_actor/actor.rs | 12 ++--- .../src/index_controller/index_actor/error.rs | 5 +- .../src/index_controller/index_actor/mod.rs | 10 +--- .../src/index_controller/index_actor/store.rs | 3 +- meilisearch-http/tests/stats/mod.rs | 2 - 13 files changed, 101 insertions(+), 103 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 757a2a9e8..5cb6c28bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1708,7 +1708,7 @@ dependencies = [ [[package]] name = "milli" version = "0.4.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.4.0#3bd4cf94cc60733393b94021fca77eb100bfe17a" +source = "git+https://github.com/meilisearch/milli.git?rev=70bee7d405711d5e6d24b62710e92671be5ac67a#70bee7d405711d5e6d24b62710e92671be5ac67a" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 70ab30d34..5ba376f08 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.4.0" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "70bee7d405711d5e6d24b62710e92671be5ac67a" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 7468f0630..5bf41231f 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -7,6 +7,7 @@ use actix_web::body::Body; use actix_web::dev::BaseHttpResponseBuilder; use actix_web::http::StatusCode; use meilisearch_error::{Code, ErrorCode}; +use milli::UserError; use serde::ser::{Serialize, SerializeStruct, Serializer}; use crate::index_controller::error::IndexControllerError; @@ -139,3 +140,44 @@ macro_rules! internal_error { )* } } + +#[derive(Debug)] +pub struct MilliError<'a>(pub &'a milli::Error); + +impl Error for MilliError<'_> {} + +impl fmt::Display for MilliError<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl ErrorCode for MilliError<'_> { + fn error_code(&self) -> Code { + match self.0 { + milli::Error::InternalError(_) => Code::Internal, + milli::Error::IoError(_) => Code::Internal, + milli::Error::UserError(ref error) => { + match error { + // TODO: wait for spec for new error codes. + UserError::AttributeLimitReached + | UserError::Csv(_) + | UserError::SerdeJson(_) + | UserError::MaxDatabaseSizeReached + | UserError::InvalidCriterionName { .. } + | UserError::InvalidDocumentId { .. } + | UserError::InvalidStoreFile + | UserError::NoSpaceLeftOnDevice + | UserError::DocumentLimitReached => todo!(), + UserError::InvalidFilter(_) => Code::Filter, + UserError::InvalidFilterAttribute(_) => Code::Filter, + UserError::MissingDocumentId { .. } => Code::MissingDocumentId, + UserError::MissingPrimaryKey => Code::MissingPrimaryKey, + UserError::PrimaryKeyCannotBeChanged => Code::PrimaryKeyAlreadyPresent, + UserError::PrimaryKeyCannotBeReset => Code::PrimaryKeyAlreadyPresent, + UserError::UnknownInternalDocumentId { .. } => Code::DocumentNotFound, + } + }, + } + } +} diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index f66edd9d1..92513d10f 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::option::IndexerOpts; -use super::error::{IndexError, Result}; +use super::error::Result; use super::{update_handler::UpdateHandler, Index, Settings, Unchecked}; #[derive(Serialize, Deserialize)] @@ -38,9 +38,7 @@ impl Index { let document_file_path = path.as_ref().join(DATA_FILE_NAME); let mut document_file = File::create(&document_file_path)?; - let documents = self - .all_documents(txn) - .map_err(|e| IndexError::Internal(e.into()))?; + let documents = self.all_documents(txn)?; let fields_ids_map = self.fields_ids_map(txn)?; // dump documents diff --git a/meilisearch-http/src/index/error.rs b/meilisearch-http/src/index/error.rs index 32f13a143..17b0ef2be 100644 --- a/meilisearch-http/src/index/error.rs +++ b/meilisearch-http/src/index/error.rs @@ -3,6 +3,8 @@ use std::error::Error; use meilisearch_error::{Code, ErrorCode}; use serde_json::Value; +use crate::error::MilliError; + pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] @@ -13,6 +15,8 @@ pub enum IndexError { DocumentNotFound(String), #[error("error with facet: {0}")] Facet(#[from] FacetError), + #[error("{0}")] + Milli(#[from] milli::Error), } internal_error!( @@ -29,6 +33,7 @@ impl ErrorCode for IndexError { IndexError::Internal(_) => Code::Internal, IndexError::DocumentNotFound(_) => Code::DocumentNotFound, IndexError::Facet(e) => e.error_code(), + IndexError::Milli(e) => MilliError(e).error_code(), } } } diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 816629524..ca1518a2e 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -51,8 +51,7 @@ impl Index { create_dir_all(&path)?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = - milli::Index::new(options, &path).map_err(|e| IndexError::Internal(e.into()))?; + let index = milli::Index::new(options, &path)?; Ok(Index(Arc::new(index))) } @@ -70,11 +69,7 @@ impl Index { .searchable_fields(&txn)? .map(|fields| fields.into_iter().map(String::from).collect()); - let faceted_attributes = self - .faceted_fields(&txn) - .map_err(|e| IndexError::Internal(Box::new(e)))? - .into_iter() - .collect(); + let faceted_attributes = self.faceted_fields(&txn)?.into_iter().collect(); let criteria = self .criteria(&txn)? @@ -83,8 +78,7 @@ impl Index { .collect(); let stop_words = self - .stop_words(&txn) - .map_err(|e| IndexError::Internal(e.into()))? + .stop_words(&txn)? .map(|stop_words| -> Result> { Ok(stop_words.stream().into_strs()?.into_iter().collect()) }) @@ -126,9 +120,8 @@ impl Index { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = self - .fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map) - .map_err(|e| IndexError::Internal(e.into()))?; + let fields_to_display = + self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -136,8 +129,7 @@ impl Index { for entry in iter { let (_id, obkv) = entry?; - let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv) - .map_err(|e| IndexError::Internal(e.into()))?; + let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?; documents.push(object); } @@ -153,31 +145,24 @@ impl Index { let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = self - .fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map) - .map_err(|e| IndexError::Internal(e.into()))?; + let fields_to_display = + self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; let internal_id = self - .external_documents_ids(&txn) - .map_err(|e| IndexError::Internal(e.into()))? + .external_documents_ids(&txn)? .get(doc_id.as_bytes()) .ok_or_else(|| IndexError::DocumentNotFound(doc_id.clone()))?; let document = self - .documents(&txn, std::iter::once(internal_id)) - .map_err(|e| IndexError::Internal(e.into()))? + .documents(&txn, std::iter::once(internal_id))? .into_iter() .next() - .map(|(_, d)| d); + .map(|(_, d)| d) + .ok_or(IndexError::DocumentNotFound(doc_id))?; - match document { - Some(document) => { - let document = obkv_to_json(&fields_to_display, &fields_ids_map, document) - .map_err(|e| IndexError::Internal(e.into()))?; - Ok(document) - } - None => Err(IndexError::DocumentNotFound(doc_id)), - } + let document = obkv_to_json(&fields_to_display, &fields_ids_map, document)?; + + Ok(document) } pub fn size(&self) -> u64 { @@ -190,8 +175,7 @@ impl Index { attributes_to_retrieve: &Option>, fields_ids_map: &milli::FieldsIdsMap, ) -> Result> { - let mut displayed_fields_ids = match self.displayed_fields_ids(&txn) - .map_err(|e| IndexError::Internal(Box::new(e)))? { + let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { Some(ids) => ids.into_iter().collect::>(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 3122f784e..c18a0bbb4 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -12,7 +12,7 @@ use serde_json::Value; use crate::index::error::FacetError; -use super::error::{IndexError, Result}; +use super::error::Result; use super::Index; pub type Document = IndexMap; @@ -97,9 +97,8 @@ impl Index { matching_words, candidates, .. - } = search - .execute() - .map_err(|e| IndexError::Internal(e.into()))?; + } = search.execute()?; + let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); let displayed_ids = self @@ -164,6 +163,8 @@ impl Index { let mut documents = Vec::new(); + let documents_iter = self.documents(&rtxn, documents_ids)?; + for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?; let formatted = format_fields( @@ -191,8 +192,7 @@ impl Index { } let distribution = facet_distribution .candidates(candidates) - .execute() - .map_err(|e| IndexError::Internal(e.into()))?; + .execute()?; Some(distribution) } @@ -528,8 +528,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { fn parse_filter(facets: &Value, index: &Index, txn: &RoTxn) -> Result> { match facets { Value::String(expr) => { - let condition = FilterCondition::from_str(txn, index, expr) - .map_err(|e| IndexError::Internal(e.into()))?; + let condition = FilterCondition::from_str(txn, index, expr)?; Ok(Some(condition)) } Value::Array(arr) => parse_filter_array(txn, index, arr), @@ -566,7 +565,7 @@ fn parse_filter_array( } } - FilterCondition::from_array(txn, &index.0, ands).map_err(|e| IndexError::Internal(Box::new(e))) + Ok(FilterCondition::from_array(txn, &index.0, ands)?) } #[cfg(test)] diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 84fc945d5..5d9c82a83 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -8,7 +8,6 @@ use log::info; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize, Serializer}; -use crate::index::error::IndexError; use crate::index_controller::UpdateResult; use super::error::Result; @@ -206,11 +205,9 @@ impl Index { // Set the primary key if not set already, ignore if already set. if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) { - let mut builder = UpdateBuilder::new(0) - .settings(txn, &self); + let mut builder = UpdateBuilder::new(0).settings(txn, &self); builder.set_primary_key(primary_key.to_string()); - builder.execute(|_, _| ()) - .map_err(|e| IndexError::Internal(Box::new(e)))?; + builder.execute(|_, _| ())?; } let mut builder = update_builder.index_documents(txn, self); @@ -222,15 +219,9 @@ impl Index { let gzipped = false; let addition = match content { - Some(content) if gzipped => builder - .execute(GzDecoder::new(content), indexing_callback) - .map_err(|e| IndexError::Internal(e.into()))?, - Some(content) => builder - .execute(content, indexing_callback) - .map_err(|e| IndexError::Internal(e.into()))?, - None => builder - .execute(std::io::empty(), indexing_callback) - .map_err(|e| IndexError::Internal(e.into()))?, + Some(content) if gzipped => builder.execute(GzDecoder::new(content), indexing_callback)?, + Some(content) => builder.execute(content, indexing_callback)?, + None => builder.execute(std::io::empty(), indexing_callback)?, }; info!("document addition done: {:?}", addition); @@ -243,13 +234,11 @@ impl Index { let mut wtxn = self.write_txn()?; let builder = update_builder.clear_documents(&mut wtxn, self); - match builder.execute() { - Ok(_count) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(IndexError::Internal(Box::new(e))), - } + let _count = builder.execute()?; + + wtxn.commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into) } pub fn update_settings_txn<'a, 'b>( @@ -308,9 +297,7 @@ impl Index { } } - builder - .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)) - .map_err(|e| IndexError::Internal(e.into()))?; + builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step))?; Ok(UpdateResult::Other) } @@ -332,22 +319,17 @@ impl Index { update_builder: UpdateBuilder, ) -> Result { let mut txn = self.write_txn()?; - let mut builder = update_builder - .delete_documents(&mut txn, self) - .map_err(|e| IndexError::Internal(e.into()))?; + let mut builder = update_builder.delete_documents(&mut txn, self)?; // We ignore unexisting document ids document_ids.iter().for_each(|id| { builder.delete_external_id(id); }); - match builder.execute() { - Ok(deleted) => txn - .commit() - .and(Ok(UpdateResult::DocumentDeletion { deleted })) - .map_err(Into::into), - Err(e) => Err(IndexError::Internal(Box::new(e))), - } + let deleted = builder.execute()?; + txn.commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into) } } diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 28c783eb2..f7b1a0981 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -268,9 +268,7 @@ impl IndexActor { } let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); builder.set_primary_key(primary_key); - builder - .execute(|_, _| ()) - .map_err(|e| IndexActorError::Internal(Box::new(e)))?; + builder.execute(|_, _| ())?; let meta = IndexMeta::new_txn(&index, &txn)?; txn.commit()?; Ok(meta) @@ -340,13 +338,9 @@ impl IndexActor { Ok(IndexStats { size: index.size(), - number_of_documents: index - .number_of_documents(&rtxn) - .map_err(|e| IndexActorError::Internal(Box::new(e)))?, + number_of_documents: index.number_of_documents(&rtxn)?, is_indexing: None, - fields_distribution: index - .fields_distribution(&rtxn) - .map_err(|e| IndexActorError::Internal(e.into()))?, + fields_distribution: index.fields_distribution(&rtxn)?, }) }) .await? diff --git a/meilisearch-http/src/index_controller/index_actor/error.rs b/meilisearch-http/src/index_controller/index_actor/error.rs index 7c7ad1607..37b43980c 100644 --- a/meilisearch-http/src/index_controller/index_actor/error.rs +++ b/meilisearch-http/src/index_controller/index_actor/error.rs @@ -1,6 +1,6 @@ use meilisearch_error::{Code, ErrorCode}; -use crate::index::error::IndexError; +use crate::{error::MilliError, index::error::IndexError}; pub type Result = std::result::Result; @@ -16,6 +16,8 @@ pub enum IndexActorError { ExistingPrimaryKey, #[error("Internal Index Error: {0}")] Internal(Box), + #[error("{0}")] + Milli(#[from] milli::Error), } macro_rules! internal_error { @@ -40,6 +42,7 @@ impl ErrorCode for IndexActorError { IndexActorError::UnexistingIndex => Code::IndexNotFound, IndexActorError::ExistingPrimaryKey => Code::PrimaryKeyAlreadyPresent, IndexActorError::Internal(_) => Code::Internal, + IndexActorError::Milli(e) => MilliError(e).error_code(), } } } diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 65f196e41..4085dc0bd 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -17,8 +17,6 @@ use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings use crate::index_controller::{Failed, IndexStats, Processed, Processing}; use error::Result; -use self::error::IndexActorError; - use super::IndexSettings; mod actor; @@ -42,12 +40,8 @@ impl IndexMeta { } fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { - let created_at = index - .created_at(&txn) - .map_err(|e| IndexActorError::Internal(Box::new(e)))?; - let updated_at = index - .updated_at(&txn) - .map_err(|e| IndexActorError::Internal(Box::new(e)))?; + let created_at = index.created_at(&txn)?; + let updated_at = index.updated_at(&txn)?; let primary_key = index.primary_key(&txn)?.map(String::from); Ok(Self { created_at, diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs index 4c2aed622..2cfda61b5 100644 --- a/meilisearch-http/src/index_controller/index_actor/store.rs +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -61,8 +61,7 @@ impl IndexStore for MapIndexStore { let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); builder.set_primary_key(primary_key); - builder.execute(|_, _| ()) - .map_err(|e| IndexActorError::Internal(Box::new(e)))?; + builder.execute(|_, _| ())?; txn.commit()?; } diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index 130849a64..b408823de 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -39,8 +39,6 @@ async fn stats() { assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 0); assert!(response["indexes"]["test"]["isIndexing"] == false); - let last_update = response["lastUpdate"].as_str().unwrap(); - let documents = json!([ { "id": 1, From fa573dabf057022a08c9ea0049078bf7b7e3ae4c Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 17 Jun 2021 14:38:52 +0200 Subject: [PATCH 446/522] fmt --- meilisearch-http/src/error.rs | 2 +- meilisearch-http/src/index/error.rs | 3 +-- meilisearch-http/src/index/search.rs | 6 ++---- meilisearch-http/src/index/updates.rs | 8 ++++++-- meilisearch-http/src/index_controller/snapshot.rs | 6 ++++-- .../src/index_controller/update_actor/error.rs | 4 ++-- .../src/index_controller/uuid_resolver/actor.rs | 2 +- .../src/index_controller/uuid_resolver/mod.rs | 3 +-- .../src/index_controller/uuid_resolver/store.rs | 2 +- meilisearch-http/tests/stats/mod.rs | 1 - 10 files changed, 19 insertions(+), 18 deletions(-) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 5bf41231f..f715399ff 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -177,7 +177,7 @@ impl ErrorCode for MilliError<'_> { UserError::PrimaryKeyCannotBeReset => Code::PrimaryKeyAlreadyPresent, UserError::UnknownInternalDocumentId { .. } => Code::DocumentNotFound, } - }, + } } } } diff --git a/meilisearch-http/src/index/error.rs b/meilisearch-http/src/index/error.rs index 17b0ef2be..7692c57f0 100644 --- a/meilisearch-http/src/index/error.rs +++ b/meilisearch-http/src/index/error.rs @@ -20,8 +20,7 @@ pub enum IndexError { } internal_error!( - IndexError: - std::io::Error, + IndexError: std::io::Error, heed::Error, fst::Error, serde_json::Error diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index c18a0bbb4..df3890635 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -190,9 +190,7 @@ impl Index { if fields.iter().all(|f| f != "*") { facet_distribution.facets(fields); } - let distribution = facet_distribution - .candidates(candidates) - .execute()?; + let distribution = facet_distribution.candidates(candidates).execute()?; Some(distribution) } @@ -532,7 +530,7 @@ fn parse_filter(facets: &Value, index: &Index, txn: &RoTxn) -> Result parse_filter_array(txn, index, arr), - v => return Err(FacetError::InvalidExpression(&["Array"], v.clone()).into()), + v => Err(FacetError::InvalidExpression(&["Array"], v.clone()).into()), } } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 5d9c82a83..6ace7e375 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -219,7 +219,9 @@ impl Index { let gzipped = false; let addition = match content { - Some(content) if gzipped => builder.execute(GzDecoder::new(content), indexing_callback)?, + Some(content) if gzipped => { + builder.execute(GzDecoder::new(content), indexing_callback)? + } Some(content) => builder.execute(content, indexing_callback)?, None => builder.execute(std::io::empty(), indexing_callback)?, }; @@ -297,7 +299,9 @@ impl Index { } } - builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step))?; + builder.execute(|indexing_step, update_id| { + info!("update {}: {:?}", update_id, indexing_step) + })?; Ok(UpdateResult::Other) } diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index b3bfca2db..ed2d6cb8a 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -141,9 +141,11 @@ mod test { use super::*; use crate::index_controller::index_actor::MockIndexActorHandle; use crate::index_controller::update_actor::{ - MockUpdateActorHandle, UpdateActorHandleImpl, error::UpdateActorError, + error::UpdateActorError, MockUpdateActorHandle, UpdateActorHandleImpl, + }; + use crate::index_controller::uuid_resolver::{ + error::UuidResolverError, MockUuidResolverHandle, }; - use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, error::UuidResolverError}; #[actix_rt::test] async fn test_normal() { diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs index 003d90054..a78e6576f 100644 --- a/meilisearch-http/src/index_controller/update_actor/error.rs +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -7,6 +7,7 @@ use crate::index_controller::index_actor::error::IndexActorError; pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] +#[allow(clippy::large_enum_variant)] pub enum UpdateActorError { #[error("Update {0} doesn't exist.")] UnexistingUpdate(u64), @@ -33,8 +34,7 @@ impl From for UpdateActorError { } internal_error!( - UpdateActorError: - heed::Error, + UpdateActorError: heed::Error, std::io::Error, serde_json::Error, actix_http::error::PayloadError, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index 2c73aa490..efad15c92 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -4,7 +4,7 @@ use log::{info, warn}; use tokio::sync::mpsc; use uuid::Uuid; -use super::{Result, UuidResolveMsg, error::UuidResolverError, UuidStore}; +use super::{error::UuidResolverError, Result, UuidResolveMsg, UuidStore}; pub struct UuidResolverActor { inbox: mpsc::Receiver, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index b7e40618e..da6c1264d 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -10,9 +10,9 @@ use std::path::PathBuf; use uuid::Uuid; use actor::UuidResolverActor; +use error::Result; use message::UuidResolveMsg; use store::UuidStore; -use error::Result; #[cfg(test)] use mockall::automock; @@ -22,7 +22,6 @@ pub use store::HeedUuidStore; const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB - #[async_trait::async_trait] #[cfg_attr(test, automock)] pub trait UuidResolverHandle { diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 9107ce908..f02d22d7f 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -8,7 +8,7 @@ use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use super::{Result, error::UuidResolverError, UUID_STORE_SIZE}; +use super::{error::UuidResolverError, Result, UUID_STORE_SIZE}; use crate::helpers::EnvSizer; #[derive(Serialize, Deserialize)] diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index b408823de..5a065d1d2 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -60,7 +60,6 @@ async fn stats() { index.wait_update_id(0).await; - let (response, code) = server.stats().await; assert_eq!(code, 200); From 0bfdf9a7856d72bca041e3eb94fe8dedb3207a9c Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 17 Jun 2021 16:42:53 +0200 Subject: [PATCH 447/522] bump milli --- Cargo.lock | 4 ++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5cb6c28bf..75102b14e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1707,8 +1707,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.4.0" -source = "git+https://github.com/meilisearch/milli.git?rev=70bee7d405711d5e6d24b62710e92671be5ac67a#70bee7d405711d5e6d24b62710e92671be5ac67a" +version = "0.4.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.4.1#a67ccfdf3ac093b51bdf5ada3621fd6663897497" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 5ba376f08..5e49bba32 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "70bee7d405711d5e6d24b62710e92671be5ac67a" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.4.1" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" From 763ee521be4f6145322d32498162b158d69302bd Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 21 Jun 2021 12:09:59 +0200 Subject: [PATCH 448/522] fix rebase errors --- meilisearch-http/src/index/search.rs | 3 +-- meilisearch-http/src/routes/search.rs | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index df3890635..1cdac6351 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -103,7 +103,6 @@ impl Index { let displayed_ids = self .displayed_fields_ids(&rtxn)? - .map_err(|e| IndexError::Internal(Box::new(e)))? .map(|fields| fields.into_iter().collect::>()) .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect()); @@ -165,7 +164,7 @@ impl Index { let documents_iter = self.documents(&rtxn, documents_ids)?; - for (_id, obkv) in self.documents(&rtxn, documents_ids)? { + for (_id, obkv) in documents_iter { let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?; let formatted = format_fields( &fields_ids_map, diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index af557ae95..e8b197caa 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -1,5 +1,4 @@ use std::collections::{BTreeSet, HashSet}; -use std::convert::{TryFrom, TryInto}; use actix_web::{get, post, web, HttpResponse}; use serde::Deserialize; From 56686dee40da67fe561374fec5b7851eb6e28c94 Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 21 Jun 2021 13:57:32 +0200 Subject: [PATCH 449/522] review changes --- meilisearch-http/src/error.rs | 4 ++-- meilisearch-http/src/index/dump.rs | 4 ++-- meilisearch-http/src/index/error.rs | 6 +++--- meilisearch-http/src/index_controller/dump_actor/error.rs | 2 +- meilisearch-http/src/index_controller/error.rs | 2 +- .../src/index_controller/index_actor/error.rs | 6 +++--- meilisearch-http/src/index_controller/mod.rs | 4 ++-- meilisearch-http/src/index_controller/snapshot.rs | 5 +++-- .../src/index_controller/update_actor/error.rs | 6 +++--- .../src/index_controller/uuid_resolver/error.rs | 8 ++++---- 10 files changed, 24 insertions(+), 23 deletions(-) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index f715399ff..161aa6e8c 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -14,9 +14,9 @@ use crate::index_controller::error::IndexControllerError; #[derive(Debug, thiserror::Error)] pub enum AuthenticationError { - #[error("You must have an authorization token")] + #[error("you must have an authorization token")] MissingAuthorizationHeader, - #[error("Invalid API key")] + #[error("invalid API key")] InvalidToken(String), } diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 92513d10f..52a5cabd9 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -3,7 +3,7 @@ use std::io::{BufRead, BufReader, Write}; use std::path::Path; use std::sync::Arc; -use anyhow::Context; +use anyhow::{Context, bail}; use heed::RoTxn; use indexmap::IndexMap; use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; @@ -126,7 +126,7 @@ impl Index { match Arc::try_unwrap(index.0) { Ok(inner) => inner.prepare_for_closing().wait(), - Err(_) => todo!("Could not close index properly."), + Err(_) => bail!("Could not close index properly."), } Ok(()) diff --git a/meilisearch-http/src/index/error.rs b/meilisearch-http/src/index/error.rs index 7692c57f0..b9bf71a3b 100644 --- a/meilisearch-http/src/index/error.rs +++ b/meilisearch-http/src/index/error.rs @@ -9,9 +9,9 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum IndexError { - #[error("Internal error: {0}")] + #[error("internal error: {0}")] Internal(Box), - #[error("Document with id {0} not found.")] + #[error("document with id {0} not found.")] DocumentNotFound(String), #[error("error with facet: {0}")] Facet(#[from] FacetError), @@ -39,7 +39,7 @@ impl ErrorCode for IndexError { #[derive(Debug, thiserror::Error)] pub enum FacetError { - #[error("Invalid facet expression, expected {}, found: {1}", .0.join(", "))] + #[error("invalid facet expression, expected {}, found: {1}", .0.join(", "))] InvalidExpression(&'static [&'static str], Value), } diff --git a/meilisearch-http/src/index_controller/dump_actor/error.rs b/meilisearch-http/src/index_controller/dump_actor/error.rs index 778b6b25c..ed693d5f3 100644 --- a/meilisearch-http/src/index_controller/dump_actor/error.rs +++ b/meilisearch-http/src/index_controller/dump_actor/error.rs @@ -11,7 +11,7 @@ pub enum DumpActorError { DumpAlreadyRunning, #[error("dump `{0}` does not exist")] DumpDoesNotExist(String), - #[error("Internal error: {0}")] + #[error("internal error: {0}")] Internal(Box), #[error("error while dumping uuids: {0}")] UuidResolver(#[from] UuidResolverError), diff --git a/meilisearch-http/src/index_controller/error.rs b/meilisearch-http/src/index_controller/error.rs index 0862c299e..956a755ab 100644 --- a/meilisearch-http/src/index_controller/error.rs +++ b/meilisearch-http/src/index_controller/error.rs @@ -10,7 +10,7 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum IndexControllerError { - #[error("Missing index uid")] + #[error("missing index uid")] MissingUid, #[error("index resolution error: {0}")] Uuid(#[from] UuidResolverError), diff --git a/meilisearch-http/src/index_controller/index_actor/error.rs b/meilisearch-http/src/index_controller/index_actor/error.rs index 37b43980c..244797234 100644 --- a/meilisearch-http/src/index_controller/index_actor/error.rs +++ b/meilisearch-http/src/index_controller/index_actor/error.rs @@ -10,11 +10,11 @@ pub enum IndexActorError { IndexError(#[from] IndexError), #[error("index already exists")] IndexAlreadyExists, - #[error("Index doesn't exists")] + #[error("index doesn't exists")] UnexistingIndex, - #[error("Existing primary key")] + #[error("existing primary key")] ExistingPrimaryKey, - #[error("Internal Index Error: {0}")] + #[error("internal Index Error: {0}")] Internal(Box), #[error("{0}")] Milli(#[from] milli::Error), diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index d82eb3d30..1d1a8f1cc 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -330,10 +330,10 @@ impl IndexController { pub async fn update_index( &self, uid: String, - index_settings: IndexSettings, + mut index_settings: IndexSettings, ) -> Result { if index_settings.uid.is_some() { - todo!("Can't change the index uid.") + index_settings.uid.take(); } let uuid = self.uuid_resolver.get(uid.clone()).await?; diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index ed2d6cb8a..9f0c5c0ba 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; +use anyhow::bail; use log::{error, info}; use tokio::fs; use tokio::task::spawn_blocking; @@ -108,7 +109,7 @@ pub fn load_snapshot( } } } else if db_path.as_ref().exists() && !ignore_snapshot_if_db_exists { - todo!( + bail!( "database already exists at {:?}, try to delete it or rename it", db_path .as_ref() @@ -116,7 +117,7 @@ pub fn load_snapshot( .unwrap_or_else(|_| db_path.as_ref().to_owned()) ) } else if !snapshot_path.as_ref().exists() && !ignore_missing_snapshot { - todo!( + bail!( "snapshot doesn't exist at {:?}", snapshot_path .as_ref() diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs index a78e6576f..9324c424b 100644 --- a/meilisearch-http/src/index_controller/update_actor/error.rs +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -9,14 +9,14 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] #[allow(clippy::large_enum_variant)] pub enum UpdateActorError { - #[error("Update {0} doesn't exist.")] + #[error("update {0} doesn't exist.")] UnexistingUpdate(u64), - #[error("Internal error processing update: {0}")] + #[error("internal error processing update: {0}")] Internal(Box), #[error("error with index: {0}")] IndexActor(#[from] IndexActorError), #[error( - "Update store was shut down due to a fatal error, please check your logs for more info." + "update store was shut down due to a fatal error, please check your logs for more info." )] FatalUpdateStoreError, } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/error.rs b/meilisearch-http/src/index_controller/uuid_resolver/error.rs index 30cde08c1..3d7fb8444 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/error.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/error.rs @@ -4,13 +4,13 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum UuidResolverError { - #[error("Name already exist.")] + #[error("name already exist.")] NameAlreadyExist, - #[error("Index \"{0}\" doesn't exist.")] + #[error("index \"{0}\" doesn't exist.")] UnexistingIndex(String), - #[error("Badly formatted index uid: {0}")] + #[error("badly formatted index uid: {0}")] BadlyFormatted(String), - #[error("Internal error resolving index uid: {0}")] + #[error("internal error resolving index uid: {0}")] Internal(Box), } From 2097554c09393809aae724efb82c1e1bb21948ed Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 16 Jun 2021 19:50:15 +0200 Subject: [PATCH 450/522] fix the cli --- meilisearch-http/src/index_controller/mod.rs | 8 ++++---- meilisearch-http/src/option.rs | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 0c801558b..ab244d7b2 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -82,8 +82,8 @@ pub struct Stats { impl IndexController { pub fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { - let index_size = options.max_mdb_size.get_bytes() as usize; - let update_store_size = options.max_udb_size.get_bytes() as usize; + let index_size = options.max_index_size.get_bytes() as usize; + let update_store_size = options.max_index_size.get_bytes() as usize; if let Some(ref path) = options.import_snapshot { info!("Loading from snapshot {:?}", path); @@ -97,7 +97,7 @@ impl IndexController { load_dump( &options.db_path, src_path, - options.max_mdb_size.get_bytes() as usize, + options.max_index_size.get_bytes() as usize, options.max_udb_size.get_bytes() as usize, &options.indexer_options, )?; @@ -116,7 +116,7 @@ impl IndexController { &options.dumps_dir, uuid_resolver.clone(), update_handle.clone(), - options.max_mdb_size.get_bytes() as usize, + options.max_index_size.get_bytes() as usize, options.max_udb_size.get_bytes() as usize, )?; diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 8f925bad8..d78bfd37e 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -109,15 +109,15 @@ pub struct Opt { pub no_analytics: bool, /// The maximum size, in bytes, of the main lmdb database directory - #[structopt(long, env = "MEILI_MAX_MDB_SIZE", default_value = "100 GiB")] - pub max_mdb_size: Byte, + #[structopt(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")] + pub max_index_size: Byte, /// The maximum size, in bytes, of the update lmdb database directory - #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "10 GiB")] + #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "100 GiB")] pub max_udb_size: Byte, /// The maximum size, in bytes, of accepted JSON payloads - #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "10 MiB")] + #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")] pub http_payload_size_limit: Byte, /// Read server certificates from CERTFILE. @@ -184,7 +184,7 @@ pub struct Opt { #[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")] pub dumps_dir: PathBuf, - /// Import a dump from the specified path, must be a `.tar.gz` file. + /// Import a dump from the specified path, must be a `.dump` file. #[structopt(long, conflicts_with = "import-snapshot")] pub import_dump: Option, From 1cf9f43dfe4dedecdbd86e12827afe8144be9337 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 16 Jun 2021 20:48:37 +0200 Subject: [PATCH 451/522] fix the tests --- meilisearch-http/tests/common/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index bf6c95474..cd9dfb2d0 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -73,7 +73,7 @@ pub fn default_settings(dir: impl AsRef) -> Opt { env: "development".to_owned(), #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics: true, - max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), + max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), ssl_cert_path: None, From cf94b8e6e0d8cc1c6ff5be96c23081a3e4edbf91 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 21 Jun 2021 17:36:54 +0200 Subject: [PATCH 452/522] run cargo flaky only 100 times --- .github/workflows/flaky.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml index 88c8ee6ca..570bc532e 100644 --- a/.github/workflows/flaky.yml +++ b/.github/workflows/flaky.yml @@ -11,5 +11,5 @@ jobs: - uses: actions/checkout@v2 - name: Install cargo-flaky run: cargo install cargo-flaky - - name: Run cargo flaky 1000 times - run: cargo flaky -i 1000 --release + - name: Run cargo flaky 100 times + run: cargo flaky -i 100 --release From 76727455ca94985d1f88c2a4b7dbc052fb64bf73 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 21 Jun 2021 18:13:00 +0200 Subject: [PATCH 453/522] ignore all the options related to the indexer --- meilisearch-http/src/option.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index d78bfd37e..8cf2e0aba 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -188,7 +188,7 @@ pub struct Opt { #[structopt(long, conflicts_with = "import-snapshot")] pub import_dump: Option, - #[structopt(flatten)] + #[structopt(skip)] pub indexer_options: IndexerOpts, } From 1e4592dd7e0dead0ebb6a06b6fc2af7187b10e7b Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 21 Jun 2021 18:42:47 +0200 Subject: [PATCH 454/522] enable errors in updates --- meilisearch-http/src/error.rs | 98 ++++++------------- meilisearch-http/src/index/update_handler.rs | 2 +- .../src/index_controller/error.rs | 5 + meilisearch-http/src/index_controller/mod.rs | 2 +- .../update_actor/store/mod.rs | 2 +- .../src/index_controller/updates.rs | 12 +-- meilisearch-http/src/routes/mod.rs | 17 ++-- 7 files changed, 52 insertions(+), 86 deletions(-) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 161aa6e8c..42ddeca3a 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -1,4 +1,3 @@ -use std::error; use std::error::Error; use std::fmt; @@ -8,9 +7,7 @@ use actix_web::dev::BaseHttpResponseBuilder; use actix_web::http::StatusCode; use meilisearch_error::{Code, ErrorCode}; use milli::UserError; -use serde::ser::{Serialize, SerializeStruct, Serializer}; - -use crate::index_controller::error::IndexControllerError; +use serde::{Serialize, Deserialize}; #[derive(Debug, thiserror::Error)] pub enum AuthenticationError { @@ -29,56 +26,34 @@ impl ErrorCode for AuthenticationError { } } -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] pub struct ResponseError { - inner: Box, -} - -impl error::Error for ResponseError {} - -impl ErrorCode for ResponseError { - fn error_code(&self) -> Code { - self.inner.error_code() - } + #[serde(skip)] + code: StatusCode, + message: String, + error_code: String, + error_type: String, + error_link: String, } impl fmt::Display for ResponseError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.inner.fmt(f) + self.message.fmt(f) } } -macro_rules! response_error { - ($($other:path), *) => { - $( - impl From<$other> for ResponseError { - fn from(error: $other) -> ResponseError { - ResponseError { - inner: Box::new(error), - } - } - } - - )* - }; -} - -response_error!(IndexControllerError, AuthenticationError); - -impl Serialize for ResponseError { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let struct_name = "ResponseError"; - let field_count = 4; - - let mut state = serializer.serialize_struct(struct_name, field_count)?; - state.serialize_field("message", &self.to_string())?; - state.serialize_field("errorCode", &self.error_name())?; - state.serialize_field("errorType", &self.error_type())?; - state.serialize_field("errorLink", &self.error_url())?; - state.end() +impl From for ResponseError +where T: ErrorCode +{ + fn from(other: T) -> Self { + Self { + code: other.http_status(), + message: other.to_string(), + error_code: other.error_name(), + error_type: other.error_type(), + error_link: other.error_url(), + } } } @@ -89,7 +64,7 @@ impl aweb::error::ResponseError for ResponseError { } fn status_code(&self) -> StatusCode { - self.http_status() + self.code } } @@ -110,25 +85,6 @@ impl ErrorCode for PayloadError { } } -impl From> for ResponseError -where - E: Error + Sync + Send + 'static, -{ - fn from(other: PayloadError) -> Self { - ResponseError { - inner: Box::new(other), - } - } -} - -pub fn payload_error_handler(err: E) -> ResponseError -where - E: Error + Sync + Send + 'static, -{ - let error = PayloadError(err); - error.into() -} - macro_rules! internal_error { ($target:ty : $($other:path), *) => { $( @@ -168,7 +124,7 @@ impl ErrorCode for MilliError<'_> { | UserError::InvalidDocumentId { .. } | UserError::InvalidStoreFile | UserError::NoSpaceLeftOnDevice - | UserError::DocumentLimitReached => todo!(), + | UserError::DocumentLimitReached => Code::Internal, UserError::InvalidFilter(_) => Code::Filter, UserError::InvalidFilterAttribute(_) => Code::Filter, UserError::MissingDocumentId { .. } => Code::MissingDocumentId, @@ -181,3 +137,11 @@ impl ErrorCode for MilliError<'_> { } } } + +pub fn payload_error_handler(err: E) -> ResponseError +where + E: Error + Sync + Send + 'static, +{ + let error = PayloadError(err); + error.into() +} diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs index 10b28bcc9..9896c9391 100644 --- a/meilisearch-http/src/index/update_handler.rs +++ b/meilisearch-http/src/index/update_handler.rs @@ -86,7 +86,7 @@ impl UpdateHandler { match result { Ok(result) => Ok(meta.process(result)), - Err(e) => Err(meta.fail(e.to_string())), + Err(e) => Err(meta.fail(e.into())), } } } diff --git a/meilisearch-http/src/index_controller/error.rs b/meilisearch-http/src/index_controller/error.rs index 956a755ab..c01eb24c0 100644 --- a/meilisearch-http/src/index_controller/error.rs +++ b/meilisearch-http/src/index_controller/error.rs @@ -1,6 +1,8 @@ use meilisearch_error::Code; use meilisearch_error::ErrorCode; +use crate::index::error::IndexError; + use super::dump_actor::error::DumpActorError; use super::index_actor::error::IndexActorError; use super::update_actor::error::UpdateActorError; @@ -20,6 +22,8 @@ pub enum IndexControllerError { UpdateActor(#[from] UpdateActorError), #[error("error with dump: {0}")] DumpActor(#[from] DumpActorError), + #[error("error with index: {0}")] + IndexError(#[from] IndexError), } impl ErrorCode for IndexControllerError { @@ -30,6 +34,7 @@ impl ErrorCode for IndexControllerError { IndexControllerError::IndexActor(e) => e.error_code(), IndexControllerError::UpdateActor(e) => e.error_code(), IndexControllerError::DumpActor(e) => e.error_code(), + IndexControllerError::IndexError(e) => e.error_code(), } } } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 1d1a8f1cc..b51a6c5b0 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -30,7 +30,7 @@ use self::error::IndexControllerError; mod dump_actor; pub mod error; -mod index_actor; +pub mod index_actor; mod snapshot; mod update_actor; mod updates; diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 6aa336835..67329b054 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -332,7 +332,7 @@ impl UpdateStore { let result = match handle.block_on(index_handle.update(index_uuid, processing.clone(), file)) { Ok(result) => result, - Err(e) => Err(processing.fail(e.to_string())), + Err(e) => Err(processing.fail(e.into())), }; // Once the pending update have been successfully processed diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 296ef9b42..13ad9f7e5 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -3,9 +3,7 @@ use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::index::{Settings, Unchecked}; - -pub type UpdateError = String; +use crate::{error::ResponseError, index::{Settings, Unchecked}}; #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { @@ -116,7 +114,7 @@ impl Processing { } } - pub fn fail(self, error: UpdateError) -> Failed { + pub fn fail(self, error: ResponseError) -> Failed { Failed { from: self, error, @@ -143,12 +141,12 @@ impl Aborted { } } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Failed { #[serde(flatten)] pub from: Processing, - pub error: UpdateError, + pub error: ResponseError, pub failed_at: DateTime, } @@ -162,7 +160,7 @@ impl Failed { } } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize)] #[serde(tag = "status", rename_all = "camelCase")] pub enum UpdateStatus { Processing(Processing), diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index a896653ef..f4581ebcb 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -4,6 +4,7 @@ use actix_web::{get, HttpResponse}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use crate::error::ResponseError; use crate::index::{Settings, Unchecked}; use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus}; @@ -89,10 +90,8 @@ pub struct FailedUpdateResult { pub update_id: u64, #[serde(rename = "type")] pub update_type: UpdateType, - pub error: String, - pub error_type: String, - pub error_code: String, - pub error_link: String, + #[serde(flatten)] + pub response: ResponseError, pub duration: f64, // in seconds pub enqueued_at: DateTime, pub processed_at: DateTime, @@ -181,13 +180,13 @@ impl From for UpdateStatusResponse { // necessary since chrono::duration don't expose a f64 secs method. let duration = Duration::from_millis(duration as u64).as_secs_f64(); + let update_id = failed.id(); + let response = failed.error; + let content = FailedUpdateResult { - update_id: failed.id(), + update_id, update_type, - error: failed.error, - error_type: String::from("todo"), - error_code: String::from("todo"), - error_link: String::from("todo"), + response, duration, enqueued_at: failed.from.from.enqueued_at, processed_at: failed.failed_at, From f91a3bc6abada8c3285dc8231dc994dfd5fdac00 Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 21 Jun 2021 18:48:05 +0200 Subject: [PATCH 455/522] set error content type to json --- meilisearch-http/src/error.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 42ddeca3a..e29d95eff 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -60,7 +60,9 @@ where T: ErrorCode impl aweb::error::ResponseError for ResponseError { fn error_response(&self) -> aweb::BaseHttpResponse { let json = serde_json::to_vec(self).unwrap(); - BaseHttpResponseBuilder::new(self.status_code()).body(json) + BaseHttpResponseBuilder::new(self.status_code()) + .content_type("application/json") + .body(json) } fn status_code(&self) -> StatusCode { From 2bdaa70f31239e2ab28cf9a8f41a5975211de991 Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 21 Jun 2021 18:56:22 +0200 Subject: [PATCH 456/522] invalid update payload returns bad_request --- meilisearch-http/src/index_controller/update_actor/actor.rs | 3 ++- meilisearch-http/src/index_controller/update_actor/error.rs | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 1a894aaba..41995727a 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -173,7 +173,8 @@ where if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() { // The json file is invalid, we use Serde to get a nice error message: file.seek(SeekFrom::Start(0))?; - let _: serde_json::Value = serde_json::from_reader(file)?; + let _: serde_json::Value = serde_json::from_reader(file) + .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?; } Some(uuid) } else { diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs index 9324c424b..a95f1eafa 100644 --- a/meilisearch-http/src/index_controller/update_actor/error.rs +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -19,6 +19,8 @@ pub enum UpdateActorError { "update store was shut down due to a fatal error, please check your logs for more info." )] FatalUpdateStoreError, + #[error("invalid payload: {0}")] + InvalidPayload(Box), } impl From> for UpdateActorError { @@ -48,6 +50,7 @@ impl ErrorCode for UpdateActorError { UpdateActorError::Internal(_) => Code::Internal, UpdateActorError::IndexActor(e) => e.error_code(), UpdateActorError::FatalUpdateStoreError => Code::Internal, + UpdateActorError::InvalidPayload(_) => Code::BadRequest, } } } From 9092d35a3cf1ff6dc95a962a39952f9c61a9f7b7 Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 21 Jun 2021 19:20:04 +0200 Subject: [PATCH 457/522] fix payload error handler --- meilisearch-http/src/error.rs | 76 +++++++++++++------ .../update_actor/store/mod.rs | 4 +- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index e29d95eff..b627ca3a5 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -5,9 +5,10 @@ use actix_web as aweb; use actix_web::body::Body; use actix_web::dev::BaseHttpResponseBuilder; use actix_web::http::StatusCode; +use aweb::error::{JsonPayloadError, QueryPayloadError}; use meilisearch_error::{Code, ErrorCode}; use milli::UserError; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Debug, thiserror::Error)] pub enum AuthenticationError { @@ -44,7 +45,8 @@ impl fmt::Display for ResponseError { } impl From for ResponseError -where T: ErrorCode +where + T: ErrorCode, { fn from(other: T) -> Self { Self { @@ -70,23 +72,6 @@ impl aweb::error::ResponseError for ResponseError { } } -#[derive(Debug)] -struct PayloadError(E); - -impl fmt::Display for PayloadError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - std::fmt::Display::fmt(&self.0, f) - } -} - -impl Error for PayloadError {} - -impl ErrorCode for PayloadError { - fn error_code(&self) -> Code { - Code::Internal - } -} - macro_rules! internal_error { ($target:ty : $($other:path), *) => { $( @@ -140,10 +125,57 @@ impl ErrorCode for MilliError<'_> { } } +impl fmt::Display for PayloadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PayloadError::Json(e) => e.fmt(f), + PayloadError::Query(e) => e.fmt(f), + } + } +} + +#[derive(Debug)] +pub enum PayloadError { + Json(JsonPayloadError), + Query(QueryPayloadError), +} + +impl Error for PayloadError {} + +impl ErrorCode for PayloadError { + fn error_code(&self) -> Code { + match self { + PayloadError::Json(err) => match err { + JsonPayloadError::Deserialize(_) => Code::BadRequest, + JsonPayloadError::Overflow => Code::PayloadTooLarge, + JsonPayloadError::ContentType => Code::UnsupportedMediaType, + JsonPayloadError::Payload(_) => Code::BadRequest, + JsonPayloadError::Serialize(_) => Code::Internal, + _ => Code::Internal, + }, + PayloadError::Query(err) => match err { + QueryPayloadError::Deserialize(_) => Code::BadRequest, + _ => Code::Internal, + }, + } + } +} + +impl From for PayloadError { + fn from(other: JsonPayloadError) -> Self { + Self::Json(other) + } +} + +impl From for PayloadError { + fn from(other: QueryPayloadError) -> Self { + Self::Query(other) + } +} + pub fn payload_error_handler(err: E) -> ResponseError where - E: Error + Sync + Send + 'static, + E: Into, { - let error = PayloadError(err); - error.into() + err.into().into() } diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 67329b054..0bdf53aee 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -565,7 +565,7 @@ fn update_uuid_to_file_path(root: impl AsRef, uuid: Uuid) -> PathBuf { #[cfg(test)] mod test { use super::*; - use crate::index_controller::{index_actor::MockIndexActorHandle, UpdateResult}; + use crate::index_controller::{UpdateResult, index_actor::{MockIndexActorHandle, error::IndexActorError}}; use futures::future::ok; @@ -644,7 +644,7 @@ mod test { if processing.id() == 0 { Box::pin(ok(Ok(processing.process(UpdateResult::Other)))) } else { - Box::pin(ok(Err(processing.fail(String::from("err"))))) + Box::pin(ok(Err(processing.fail(IndexActorError::ExistingPrimaryKey.into())))) } }); From 905ace3e13428e6faa02f2b49e1d9ee090830e43 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 11:10:57 +0200 Subject: [PATCH 458/522] fix test --- meilisearch-http/tests/settings/get_settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 679f347fa..804167517 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -33,7 +33,7 @@ async fn update_settings_unknown_field() { let server = Server::new().await; let index = server.index("test"); let (_response, code) = index.update_settings(json!({"foo": 12})).await; - assert_eq!(code, 500); + assert_eq!(code, 400); } #[actix_rt::test] From d0ef1ef174e73b312dcc2dae8ebceea5eb5c19a3 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 11:58:01 +0200 Subject: [PATCH 459/522] change errors codes --- meilisearch-http/src/error.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index b627ca3a5..7d56de738 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -103,7 +103,6 @@ impl ErrorCode for MilliError<'_> { milli::Error::UserError(ref error) => { match error { // TODO: wait for spec for new error codes. - UserError::AttributeLimitReached | UserError::Csv(_) | UserError::SerdeJson(_) | UserError::MaxDatabaseSizeReached @@ -112,6 +111,7 @@ impl ErrorCode for MilliError<'_> { | UserError::InvalidStoreFile | UserError::NoSpaceLeftOnDevice | UserError::DocumentLimitReached => Code::Internal, + UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded, UserError::InvalidFilter(_) => Code::Filter, UserError::InvalidFilterAttribute(_) => Code::Filter, UserError::MissingDocumentId { .. } => Code::MissingDocumentId, @@ -146,10 +146,11 @@ impl ErrorCode for PayloadError { fn error_code(&self) -> Code { match self { PayloadError::Json(err) => match err { - JsonPayloadError::Deserialize(_) => Code::BadRequest, JsonPayloadError::Overflow => Code::PayloadTooLarge, JsonPayloadError::ContentType => Code::UnsupportedMediaType, - JsonPayloadError::Payload(_) => Code::BadRequest, + JsonPayloadError::Payload(aweb::error::PayloadError::Overflow) => Code::PayloadTooLarge, + JsonPayloadError::Deserialize(_) + | JsonPayloadError::Payload(_) => Code::BadRequest, JsonPayloadError::Serialize(_) => Code::Internal, _ => Code::Internal, }, From 2e3d85c31af1cefd3405607b10cf51165fcc2038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 21 Jun 2021 16:59:27 +0200 Subject: [PATCH 460/522] Update milli version to v0.5.0 --- Cargo.lock | 74 +++++++++---------- meilisearch-http/Cargo.toml | 2 +- .../src/index_controller/index_actor/actor.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 4 +- meilisearch-http/tests/index/stats.rs | 8 +- meilisearch-http/tests/stats/mod.rs | 6 +- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75102b14e..94a8fbcec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,7 +67,7 @@ dependencies = [ "percent-encoding", "pin-project", "pin-project-lite", - "rand 0.8.3", + "rand 0.8.4", "regex", "serde", "sha-1 0.9.6", @@ -209,9 +209,9 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "0.5.0-beta.2" +version = "0.5.0-beta.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" +checksum = "0d048c6986743105c1e8e9729fbc8d5d1667f2f62393a58be8d85a7d9a5a6c8d" dependencies = [ "proc-macro2 1.0.27", "quote 1.0.9", @@ -667,9 +667,9 @@ checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" [[package]] name = "cpufeatures" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed00c67cb5d0a7d64a44f6ad2668db7e7530311dd53ea79bcd4fb022c64911c8" +checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" dependencies = [ "libc", ] @@ -1375,9 +1375,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" [[package]] name = "itertools" @@ -1707,8 +1707,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.4.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.4.1#a67ccfdf3ac093b51bdf5ada3621fd6663897497" +version = "0.5.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.5.0#b073fd49ea04fbb8da940f6357c952b34af94c0e" dependencies = [ "bstr", "byteorder", @@ -1774,9 +1774,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.11" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" +checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" dependencies = [ "libc", "log", @@ -1930,9 +1930,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.34" +version = "0.10.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" +checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1950,9 +1950,9 @@ checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" -version = "0.9.63" +version = "0.9.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" +checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" dependencies = [ "autocfg", "cc", @@ -2301,14 +2301,14 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", "rand_chacha 0.3.1", - "rand_core 0.6.2", - "rand_hc 0.3.0", + "rand_core 0.6.3", + "rand_hc 0.3.1", ] [[package]] @@ -2328,7 +2328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.2", + "rand_core 0.6.3", ] [[package]] @@ -2357,9 +2357,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ "getrandom 0.2.3", ] @@ -2375,11 +2375,11 @@ dependencies = [ [[package]] name = "rand_hc" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" dependencies = [ - "rand_core 0.6.2", + "rand_core 0.6.3", ] [[package]] @@ -2427,9 +2427,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" +checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" dependencies = [ "bitflags", ] @@ -2468,9 +2468,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" dependencies = [ "base64", "bytes 1.0.1", @@ -2539,9 +2539,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce" +checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49" [[package]] name = "rustc_version" @@ -2735,7 +2735,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbbe485e384cb5540940e65d729820ffcbedc0c902fcb27081e44dacfe6a0c34" dependencies = [ "lazy_static", - "rand 0.8.3", + "rand 0.8.4", "sentry-types", "serde", "serde_json", @@ -3099,7 +3099,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.3", + "rand 0.8.4", "redox_syscall", "remove_dir_all", "winapi", @@ -3219,9 +3219,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.6.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975" +checksum = "5fb2ed024293bb19f7a5dc54fe83bf86532a44c12a2bb8ba40d64a4509395ca2" dependencies = [ "autocfg", "bytes 1.0.1", @@ -3445,9 +3445,9 @@ dependencies = [ [[package]] name = "vcpkg" -version = "0.2.13" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vec_map" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 5e49bba32..efe78c70f 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.4.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.5.0" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index f7b1a0981..15d96b7ad 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -340,7 +340,7 @@ impl IndexActor { size: index.size(), number_of_documents: index.number_of_documents(&rtxn)?, is_indexing: None, - fields_distribution: index.fields_distribution(&rtxn)?, + field_distribution: index.field_distribution(&rtxn)?, }) }) .await? diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index b51a6c5b0..416258555 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -7,7 +7,7 @@ use actix_web::web::{Bytes, Payload}; use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use log::info; -use milli::FieldsDistribution; +use milli::FieldDistribution; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::time::sleep; @@ -63,7 +63,7 @@ pub struct IndexStats { /// index returns it, since it is the `UpdateStore` that knows what index is currently indexing. It is /// later set to either true or false, we we retrieve the information from the `UpdateStore` pub is_indexing: Option, - pub fields_distribution: FieldsDistribution, + pub field_distribution: FieldDistribution, } #[derive(Clone)] diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs index b0df427b6..8494bbae3 100644 --- a/meilisearch-http/tests/index/stats.rs +++ b/meilisearch-http/tests/index/stats.rs @@ -15,7 +15,7 @@ async fn stats() { assert_eq!(code, 200); assert_eq!(response["numberOfDocuments"], 0); assert!(response["isIndexing"] == false); - assert!(response["fieldsDistribution"] + assert!(response["fieldDistribution"] .as_object() .unwrap() .is_empty()); @@ -42,7 +42,7 @@ async fn stats() { assert_eq!(code, 200); assert_eq!(response["numberOfDocuments"], 2); assert!(response["isIndexing"] == false); - assert_eq!(response["fieldsDistribution"]["id"], 2); - assert_eq!(response["fieldsDistribution"]["name"], 1); - assert_eq!(response["fieldsDistribution"]["age"], 1); + assert_eq!(response["fieldDistribution"]["id"], 2); + assert_eq!(response["fieldDistribution"]["name"], 1); + assert_eq!(response["fieldDistribution"]["age"], 1); } diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index 5a065d1d2..a5d08b52b 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -67,7 +67,7 @@ async fn stats() { assert!(response.get("lastUpdate").is_some()); assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 2); assert!(response["indexes"]["test"]["isIndexing"] == false); - assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["id"], 2); - assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["name"], 1); - assert_eq!(response["indexes"]["test"]["fieldsDistribution"]["age"], 1); + assert_eq!(response["indexes"]["test"]["fieldDistribution"]["id"], 2); + assert_eq!(response["indexes"]["test"]["fieldDistribution"]["name"], 1); + assert_eq!(response["indexes"]["test"]["fieldDistribution"]["age"], 1); } From e2844f3a928d5a7cc0af77267329d3d006b5cdb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 21 Jun 2021 17:00:09 +0200 Subject: [PATCH 461/522] Update tokenizer version to v0.2.3 --- Cargo.lock | 20 ++++++++++++++++++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94a8fbcec..3b99965f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1626,7 +1626,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.3", "memmap", "milli", "mime", @@ -1680,6 +1680,22 @@ dependencies = [ "whatlang", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.2.3" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "memchr" version = "2.4.0" @@ -1726,7 +1742,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.2", "memmap", "obkv", "once_cell", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index efe78c70f..62625b2f1 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -49,7 +49,7 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.5.0" } mime = "0.3.16" From 9cc31c2258e1dbc6dde592aab30c3c2ad66fd8ea Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 14:22:36 +0200 Subject: [PATCH 462/522] fix get search crop len --- meilisearch-http/src/index/mod.rs | 2 +- meilisearch-http/src/index/search.rs | 2 +- meilisearch-http/src/routes/search.rs | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index ca1518a2e..5a005fdc0 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -13,7 +13,7 @@ use serde_json::{Map, Value}; use crate::helpers::EnvSizer; use error::Result; -pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; +pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT, default_crop_length}; pub use updates::{Checked, Facets, Settings, Unchecked}; use self::error::IndexError; diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 0a658ec30..6a03bdbb7 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -23,7 +23,7 @@ const fn default_search_limit() -> usize { } pub const DEFAULT_CROP_LENGTH: usize = 200; -const fn default_crop_length() -> usize { +pub const fn default_crop_length() -> usize { DEFAULT_CROP_LENGTH } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index e8b197caa..192fd4994 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -6,7 +6,7 @@ use serde_json::Value; use crate::error::ResponseError; use crate::helpers::Authentication; -use crate::index::{SearchQuery, DEFAULT_SEARCH_LIMIT}; +use crate::index::{SearchQuery, default_crop_length, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; @@ -22,10 +22,12 @@ pub struct SearchQueryGet { limit: Option, attributes_to_retrieve: Option, attributes_to_crop: Option, + #[serde(default = "default_crop_length")] crop_length: usize, attributes_to_highlight: Option, filter: Option, - matches: Option, + #[serde(default = "Default::default")] + matches: bool, facet_distributions: Option, } @@ -64,7 +66,7 @@ impl From for SearchQuery { crop_length: other.crop_length, attributes_to_highlight, filter, - matches: other.matches, + matches: Some(other.matches), facet_distributions, } } From caa231aebe6692a79bb3ef5c2c03c887cba5e558 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 16 Jun 2021 14:52:06 +0200 Subject: [PATCH 463/522] fix race condition --- meilisearch-http/src/index_controller/mod.rs | 17 +++++++++++++++-- .../index_controller/update_actor/store/mod.rs | 13 ++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index b51a6c5b0..313709e21 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -6,6 +6,7 @@ use std::time::Duration; use actix_web::web::{Bytes, Payload}; use chrono::{DateTime, Utc}; use futures::stream::StreamExt; +use log::error; use log::info; use milli::FieldsDistribution; use serde::{Deserialize, Serialize}; @@ -256,8 +257,20 @@ impl IndexController { pub async fn delete_index(&self, uid: String) -> Result<()> { let uuid = self.uuid_resolver.delete(uid).await?; - self.update_handle.delete(uuid).await?; - self.index_handle.delete(uuid).await?; + + // We remove the index from the resolver synchronously, and effectively perform the index + // deletion as a background task. + let update_handle = self.update_handle.clone(); + let index_handle = self.index_handle.clone(); + tokio::spawn(async move { + if let Err(e) = update_handle.delete(uuid).await { + error!("Error while deleting index: {}", e); + } + if let Err(e) = index_handle.delete(uuid).await { + error!("Error while deleting index: {}", e); + } + }); + Ok(()) } diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 0bdf53aee..1ee30c99d 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -428,7 +428,8 @@ impl UpdateStore { Ok(None) } - /// Delete all updates for an index from the update store. + /// Delete all updates for an index from the update store. If the currently processing update + /// is for `index_uuid`, the call will block until the update is terminated. pub fn delete_all(&self, index_uuid: Uuid) -> Result<()> { let mut txn = self.env.write_txn()?; // Contains all the content file paths that we need to be removed if the deletion was successful. @@ -469,8 +470,14 @@ impl UpdateStore { let _ = remove_file(path); }); - // We don't care about the currently processing update, since it will be removed by itself - // once its done processing, and we can't abort a running update. + // If the currently processing update is from our index, we wait until it is + // finished before returning. This ensure that no write to the index occurs after we delete it. + if let State::Processing(uuid, _) = *self.state.read() { + if uuid == index_uuid { + // wait for a write lock, do nothing with it. + self.state.write(); + } + } Ok(()) } From 97ef4a6c225180566b328ba22add30d7b7564225 Mon Sep 17 00:00:00 2001 From: marin postma Date: Mon, 21 Jun 2021 23:38:59 +0200 Subject: [PATCH 464/522] implement matches --- meilisearch-http/src/index/search.rs | 414 +++++++++++++++++++------- meilisearch-http/src/routes/search.rs | 2 +- 2 files changed, 309 insertions(+), 107 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 6a03bdbb7..4c85063b5 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -6,7 +6,7 @@ use either::Either; use heed::RoTxn; use indexmap::IndexMap; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token}; -use milli::{FilterCondition, FieldId, FieldsIdsMap, MatchingWords}; +use milli::{FieldId, FieldsIdsMap, FilterCondition, MatchingWords}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -16,6 +16,13 @@ use super::error::Result; use super::Index; pub type Document = IndexMap; +type MatchesInfo = BTreeMap>; + +#[derive(Serialize, Debug, Clone)] +pub struct MatchInfo { + start: usize, + length: usize, +} pub const DEFAULT_SEARCH_LIMIT: usize = 20; const fn default_search_limit() -> usize { @@ -39,7 +46,9 @@ pub struct SearchQuery { #[serde(default = "default_crop_length")] pub crop_length: usize, pub attributes_to_highlight: Option>, - pub matches: Option, + // Default to false + #[serde(default = "Default::default")] + pub matches: bool, pub filter: Option, pub facet_distributions: Option>, } @@ -50,6 +59,8 @@ pub struct SearchHit { pub document: Document, #[serde(rename = "_formatted", skip_serializing_if = "Document::is_empty")] pub formatted: Document, + #[serde(rename = "_MatchesInfo", skip_serializing_if = "Option::is_none")] + pub matches_info: Option, } #[derive(Serialize)] @@ -134,13 +145,9 @@ impl Index { .cloned() .collect(); - let attr_to_highlight = query - .attributes_to_highlight - .unwrap_or_default(); + let attr_to_highlight = query.attributes_to_highlight.unwrap_or_default(); - let attr_to_crop = query - .attributes_to_crop - .unwrap_or_default(); + let attr_to_crop = query.attributes_to_crop.unwrap_or_default(); // Attributes in `formatted_options` correspond to the attributes that will be in `_formatted` // These attributes are: @@ -157,8 +164,7 @@ impl Index { ); let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut documents = Vec::new(); @@ -166,6 +172,9 @@ impl Index { for (_id, obkv) in documents_iter { let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?; + + let matches_info = query.matches.then(|| compute_matches(&matching_words, &document)); + let formatted = format_fields( &fields_ids_map, obkv, @@ -177,6 +186,7 @@ impl Index { let hit = SearchHit { document, formatted, + matches_info, }; documents.push(hit); } @@ -210,6 +220,53 @@ impl Index { } } +fn compute_matches(matcher: &impl Matcher, document: &Document) -> MatchesInfo { + let stop_words = fst::Set::default(); + let mut matches = BTreeMap::new(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + + for (key, value) in document { + let mut infos = Vec::new(); + compute_value_matches(&mut infos, value, matcher, &analyzer); + if !infos.is_empty() { + matches.insert(key.clone(), infos); + } + } + matches +} + +fn compute_value_matches<'a, A: AsRef<[u8]>>( + infos: &mut Vec, + value: &Value, + matcher: &impl Matcher, + analyzer: &Analyzer<'a, A>, +) { + match value { + Value::String(s) => { + let analyzed = analyzer.analyze(s); + let mut start = 0; + for (word, token) in analyzed.reconstruct() { + if token.is_word() { + if let Some(length) = matcher.matches(token.text()) { + infos.push(MatchInfo { start, length }); + } + } + + start += word.len(); + } + } + Value::Array(vals) => vals + .iter() + .for_each(|val| compute_value_matches(infos, val, matcher, analyzer)), + Value::Object(vals) => vals + .values() + .for_each(|val| compute_value_matches(infos, val, matcher, analyzer)), + _ => (), + } +} + fn compute_formatted_options( attr_to_highlight: &HashSet, attr_to_crop: &[String], @@ -217,8 +274,7 @@ fn compute_formatted_options( to_retrieve_ids: &BTreeSet, fields_ids_map: &FieldsIdsMap, displayed_ids: &BTreeSet, - ) -> BTreeMap { - +) -> BTreeMap { let mut formatted_options = BTreeMap::new(); add_highlight_to_formatted_options( @@ -238,10 +294,7 @@ fn compute_formatted_options( // Should not return `_formatted` if no valid attributes to highlight/crop if !formatted_options.is_empty() { - add_non_formatted_ids_to_formatted_options( - &mut formatted_options, - to_retrieve_ids, - ); + add_non_formatted_ids_to_formatted_options(&mut formatted_options, to_retrieve_ids); } formatted_options @@ -287,7 +340,7 @@ fn add_crop_to_formatted_options( Some((len, name)) => { let crop_len = len.parse::().unwrap_or(crop_length); (name, crop_len) - }, + } None => (attr.as_str(), crop_length), }; @@ -319,15 +372,13 @@ fn add_crop_to_formatted_options( fn add_non_formatted_ids_to_formatted_options( formatted_options: &mut BTreeMap, - to_retrieve_ids: &BTreeSet + to_retrieve_ids: &BTreeSet, ) { for id in to_retrieve_ids { - formatted_options - .entry(*id) - .or_insert(FormatOptions { - highlight: false, - crop: None, - }); + formatted_options.entry(*id).or_insert(FormatOptions { + highlight: false, + crop: None, + }); } } @@ -337,6 +388,7 @@ fn make_document( obkv: obkv::KvReader, ) -> Result { let mut document = Document::new(); + for attr in attributes_to_retrieve { if let Some(value) = obkv.get(*attr) { let value = serde_json::from_slice(value)?; @@ -367,11 +419,7 @@ fn format_fields>( if let Some(value) = obkv.get(*id) { let mut value: Value = serde_json::from_slice(value)?; - value = formatter.format_value( - value, - matching_words, - *format, - ); + value = formatter.format_value(value, matching_words, *format); // This unwrap must be safe since we got the ids from the fields_ids_map just // before. @@ -428,20 +476,40 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { ) -> Value { match value { Value::String(old_string) => { - let value = - self.format_string(old_string, matcher, format_options); + let value = self.format_string(old_string, matcher, format_options); Value::String(value) } Value::Array(values) => Value::Array( values .into_iter() - .map(|v| self.format_value(v, matcher, FormatOptions { highlight: format_options.highlight, crop: None })) + .map(|v| { + self.format_value( + v, + matcher, + FormatOptions { + highlight: format_options.highlight, + crop: None, + }, + ) + }) .collect(), ), Value::Object(object) => Value::Object( object .into_iter() - .map(|(k, v)| (k, self.format_value(v, matcher, FormatOptions { highlight: format_options.highlight, crop: None }))) + .map(|(k, v)| { + ( + k, + self.format_value( + v, + matcher, + FormatOptions { + highlight: format_options.highlight, + crop: None, + }, + ), + ) + }) .collect(), ), value => value, @@ -461,7 +529,9 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { let mut buffer = Vec::new(); let mut tokens = analyzed.reconstruct().peekable(); - while let Some((word, token)) = tokens.next_if(|(_, token)| matcher.matches(token.text()).is_none()) { + while let Some((word, token)) = + tokens.next_if(|(_, token)| matcher.matches(token.text()).is_none()) + { buffer.push((word, token)); } @@ -474,19 +544,16 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { }); let mut taken_after = 0; - let after_iter = tokens - .take_while(move |(word, _)| { + let after_iter = tokens.take_while(move |(word, _)| { let take = taken_after < crop_len; taken_after += word.chars().count(); take }); - let iter = before_iter - .chain(Some(token)) - .chain(after_iter); + let iter = before_iter.chain(Some(token)).chain(after_iter); Box::new(iter) - }, + } // If no word matches in the attribute None => { let mut count = 0; @@ -505,16 +572,19 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { tokens .map(|(word, token)| { + // Check if we need to do highlighting or computed matches before calling + // Matcher::match since the call is expensive. if format_options.highlight && token.is_word() { - if let Some(match_len) = matcher.matches(token.text()) { - let mut new_word = String::new(); + if let Some(length) = matcher.matches(token.text()) { + if format_options.highlight { + let mut new_word = String::new(); + new_word.push_str(&self.marks.0); + new_word.push_str(&word[..length]); + new_word.push_str(&self.marks.1); + new_word.push_str(&word[length..]); - new_word.push_str(&self.marks.0); - new_word.push_str(&word[..match_len]); - new_word.push_str(&self.marks.1); - new_word.push_str(&word[match_len..]); - - return Cow::Owned(new_word) + return Cow::Owned(new_word); + } } } Cow::Borrowed(word) @@ -573,8 +643,7 @@ mod test { #[test] fn no_ids_no_formatted() { let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let id = fields.insert("test").unwrap(); @@ -606,8 +675,7 @@ mod test { #[test] fn formatted_with_highlight_in_word() { let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -615,19 +683,39 @@ mod test { let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(title, Value::String("The Hobbit".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + title, + Value::String("The Hobbit".into()).to_string().as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(author, Value::String("J. R. R. Tolkien".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + author, + Value::String("J. R. R. Tolkien".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); let mut formatted_options = BTreeMap::new(); - formatted_options.insert(title, FormatOptions { highlight: true, crop: None }); - formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: None, + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); let mut matching_words = BTreeMap::new(); matching_words.insert("hobbit", Some(3)); @@ -648,8 +736,7 @@ mod test { #[test] fn formatted_with_crop_2() { let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -657,19 +744,39 @@ mod test { let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); let mut formatted_options = BTreeMap::new(); - formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(2) }); - formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); + formatted_options.insert( + title, + FormatOptions { + highlight: false, + crop: Some(2), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); let mut matching_words = BTreeMap::new(); matching_words.insert("potter", Some(6)); @@ -690,8 +797,7 @@ mod test { #[test] fn formatted_with_crop_10() { let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -699,19 +805,39 @@ mod test { let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); let mut formatted_options = BTreeMap::new(); - formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(10) }); - formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); + formatted_options.insert( + title, + FormatOptions { + highlight: false, + crop: Some(10), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); let mut matching_words = BTreeMap::new(); matching_words.insert("potter", Some(6)); @@ -732,8 +858,7 @@ mod test { #[test] fn formatted_with_crop_0() { let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -741,19 +866,39 @@ mod test { let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); let mut formatted_options = BTreeMap::new(); - formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(0) }); - formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); + formatted_options.insert( + title, + FormatOptions { + highlight: false, + crop: Some(0), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); let mut matching_words = BTreeMap::new(); matching_words.insert("potter", Some(6)); @@ -774,8 +919,7 @@ mod test { #[test] fn formatted_with_crop_and_no_match() { let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -783,19 +927,39 @@ mod test { let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); let mut formatted_options = BTreeMap::new(); - formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(6) }); - formatted_options.insert(author, FormatOptions { highlight: false, crop: Some(20) }); + formatted_options.insert( + title, + FormatOptions { + highlight: false, + crop: Some(6), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: Some(20), + }, + ); let mut matching_words = BTreeMap::new(); matching_words.insert("rowling", Some(3)); @@ -816,8 +980,7 @@ mod test { #[test] fn formatted_with_crop_and_highlight() { let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -825,19 +988,39 @@ mod test { let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); let mut formatted_options = BTreeMap::new(); - formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(1) }); - formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: Some(1), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); let mut matching_words = BTreeMap::new(); matching_words.insert("and", Some(3)); @@ -858,8 +1041,7 @@ mod test { #[test] fn formatted_with_crop_and_highlight_in_word() { let stop_words = fst::Set::default(); - let formatter = - Formatter::new(&stop_words, (String::from(""), String::from(""))); + let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -867,19 +1049,39 @@ mod test { let mut buf = Vec::new(); let mut obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); obkv = obkv::KvWriter::new(&mut buf); - obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes()) - .unwrap(); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); obkv.finish().unwrap(); let obkv = obkv::KvReader::new(&buf); let mut formatted_options = BTreeMap::new(); - formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(9) }); - formatted_options.insert(author, FormatOptions { highlight: false, crop: None }); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: Some(9), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); let mut matching_words = BTreeMap::new(); matching_words.insert("blood", Some(3)); diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 192fd4994..07239a257 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -66,7 +66,7 @@ impl From for SearchQuery { crop_length: other.crop_length, attributes_to_highlight, filter, - matches: Some(other.matches), + matches: other.matches.unwrap_or_default(), facet_distributions, } } From f6d1fb7ac21f0cab345dbadd51a9b4d19bb03d5e Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 10:08:38 +0200 Subject: [PATCH 465/522] fmt --- meilisearch-http/src/index/search.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 4c85063b5..b42b78d33 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -173,7 +173,9 @@ impl Index { for (_id, obkv) in documents_iter { let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?; - let matches_info = query.matches.then(|| compute_matches(&matching_words, &document)); + let matches_info = query + .matches + .then(|| compute_matches(&matching_words, &document)); let formatted = format_fields( &fields_ids_map, From c4ee937635ea61798b7674bf253078e9cf93f07c Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 10:17:39 +0200 Subject: [PATCH 466/522] optimize fromat string --- meilisearch-http/src/index/search.rs | 33 +++++++++++++--------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index b42b78d33..58132b633 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -572,26 +572,23 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { None => Box::new(analyzed.reconstruct()), }; - tokens - .map(|(word, token)| { - // Check if we need to do highlighting or computed matches before calling - // Matcher::match since the call is expensive. - if format_options.highlight && token.is_word() { - if let Some(length) = matcher.matches(token.text()) { - if format_options.highlight { - let mut new_word = String::new(); - new_word.push_str(&self.marks.0); - new_word.push_str(&word[..length]); - new_word.push_str(&self.marks.1); - new_word.push_str(&word[length..]); - - return Cow::Owned(new_word); - } + tokens.fold(String::new(), |mut out, (word, token)| { + // Check if we need to do highlighting or computed matches before calling + // Matcher::match since the call is expensive. + if format_options.highlight && token.is_word() { + if let Some(length) = matcher.matches(token.text()) { + if format_options.highlight { + out.push_str(&self.marks.0); + out.push_str(&word[..length]); + out.push_str(&self.marks.1); + out.push_str(&word[length..]); + return out; } } - Cow::Borrowed(word) - }) - .collect::() + } + out.push_str(word); + out + }) } } From eb3d63691a747018585a88be68e89a1ce75ac67c Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 11:06:30 +0200 Subject: [PATCH 467/522] add tests --- meilisearch-http/src/index/search.rs | 52 ++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 58132b633..bfe77f6af 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::time::Instant; @@ -59,7 +58,7 @@ pub struct SearchHit { pub document: Document, #[serde(rename = "_formatted", skip_serializing_if = "Document::is_empty")] pub formatted: Document, - #[serde(rename = "_MatchesInfo", skip_serializing_if = "Option::is_none")] + #[serde(rename = "_matchesInfo", skip_serializing_if = "Option::is_none")] pub matches_info: Option, } @@ -223,8 +222,8 @@ impl Index { } fn compute_matches(matcher: &impl Matcher, document: &Document) -> MatchesInfo { - let stop_words = fst::Set::default(); let mut matches = BTreeMap::new(); + let stop_words = fst::Set::default(); let mut config = AnalyzerConfig::default(); config.stop_words(&stop_words); let analyzer = Analyzer::new(config); @@ -1097,4 +1096,51 @@ mod test { assert_eq!(value["title"], "the Half-Blood Prince"); assert_eq!(value["author"], "J. K. Rowling"); } + + #[test] + fn test_compute_value_matches() { + let text = "Call me Ishmael. Some years ago—never mind how long precisely—having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world."; + let value = serde_json::json!(text); + + let mut matcher = BTreeMap::new(); + matcher.insert("ishmael", Some(3)); + matcher.insert("little", Some(6)); + matcher.insert("particular", Some(1)); + + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + + let mut infos = Vec::new(); + + compute_value_matches(&mut infos, &value, &matcher, &analyzer); + + let mut infos = infos.into_iter(); + let crop = |info: MatchInfo| &text[info.start..info.start + info.length]; + + assert_eq!(crop(infos.next().unwrap()), "Ish"); + assert_eq!(crop(infos.next().unwrap()), "little"); + assert_eq!(crop(infos.next().unwrap()), "p"); + assert_eq!(crop(infos.next().unwrap()), "little"); + assert!(infos.next().is_none()); + } + + #[test] + fn test_compute_match() { + let value = serde_json::from_str(r#"{ + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis Laboris . Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n" + }"#).unwrap(); + let mut matcher = BTreeMap::new(); + matcher.insert("green", Some(3)); + matcher.insert("mollit", Some(6)); + matcher.insert("laboris", Some(7)); + + let matches = compute_matches(&matcher, &value); + assert_eq!(format!("{:?}", matches), r##"{"about": [MatchInfo { start: 0, length: 6 }, MatchInfo { start: 31, length: 7 }, MatchInfo { start: 191, length: 7 }, MatchInfo { start: 225, length: 7 }, MatchInfo { start: 233, length: 6 }], "color": [MatchInfo { start: 0, length: 3 }]}"##); + } } From 3456a78552d3aa35b6de4a460ee5d345113c5757 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 11:27:07 +0200 Subject: [PATCH 468/522] refactor formatter share the analyzer instance between the formatter and the compute_matches function --- meilisearch-http/src/index/search.rs | 74 ++++++++++++++++++--------- meilisearch-http/src/routes/search.rs | 2 +- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index bfe77f6af..977992deb 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -163,7 +163,11 @@ impl Index { ); let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut documents = Vec::new(); @@ -174,7 +178,7 @@ impl Index { let matches_info = query .matches - .then(|| compute_matches(&matching_words, &document)); + .then(|| compute_matches(&matching_words, &document, &analyzer)); let formatted = format_fields( &fields_ids_map, @@ -221,12 +225,12 @@ impl Index { } } -fn compute_matches(matcher: &impl Matcher, document: &Document) -> MatchesInfo { +fn compute_matches>( + matcher: &impl Matcher, + document: &Document, + analyzer: &Analyzer + ) -> MatchesInfo { let mut matches = BTreeMap::new(); - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); for (key, value) in document { let mut infos = Vec::new(); @@ -455,17 +459,12 @@ impl Matcher for MatchingWords { } struct Formatter<'a, A> { - analyzer: Analyzer<'a, A>, + analyzer: &'a Analyzer<'a, A>, marks: (String, String), } impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { - pub fn new(stop_words: &'a fst::Set, marks: (String, String)) -> Self { - let mut config = AnalyzerConfig::default(); - config.stop_words(stop_words); - - let analyzer = Analyzer::new(config); - + pub fn new(analyzer: &'a Analyzer<'a, A>, marks: (String, String)) -> Self { Self { analyzer, marks } } @@ -641,7 +640,10 @@ mod test { #[test] fn no_ids_no_formatted() { let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let id = fields.insert("test").unwrap(); @@ -673,7 +675,10 @@ mod test { #[test] fn formatted_with_highlight_in_word() { let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -734,7 +739,10 @@ mod test { #[test] fn formatted_with_crop_2() { let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -795,7 +803,10 @@ mod test { #[test] fn formatted_with_crop_10() { let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -856,7 +867,10 @@ mod test { #[test] fn formatted_with_crop_0() { let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -917,7 +931,10 @@ mod test { #[test] fn formatted_with_crop_and_no_match() { let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -978,7 +995,10 @@ mod test { #[test] fn formatted_with_crop_and_highlight() { let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -1039,7 +1059,10 @@ mod test { #[test] fn formatted_with_crop_and_highlight_in_word() { let stop_words = fst::Set::default(); - let formatter = Formatter::new(&stop_words, (String::from(""), String::from(""))); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); let mut fields = FieldsIdsMap::new(); let title = fields.insert("title").unwrap(); @@ -1140,7 +1163,12 @@ mod test { matcher.insert("mollit", Some(6)); matcher.insert("laboris", Some(7)); - let matches = compute_matches(&matcher, &value); + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + + let matches = compute_matches(&matcher, &value, &analyzer); assert_eq!(format!("{:?}", matches), r##"{"about": [MatchInfo { start: 0, length: 6 }, MatchInfo { start: 31, length: 7 }, MatchInfo { start: 191, length: 7 }, MatchInfo { start: 225, length: 7 }, MatchInfo { start: 233, length: 6 }], "color": [MatchInfo { start: 0, length: 3 }]}"##); } } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 07239a257..e8f6bc86f 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -66,7 +66,7 @@ impl From for SearchQuery { crop_length: other.crop_length, attributes_to_highlight, filter, - matches: other.matches.unwrap_or_default(), + matches: other.matches, facet_distributions, } } From 0de696feafb92841fc94e34580ebf619071b88d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 22 Jun 2021 18:40:51 +0200 Subject: [PATCH 469/522] Update version for alpha 6 --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b99965f0..7b50e8f84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1592,7 +1592,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.21.0-alpha.5" +version = "0.21.0-alpha.6" dependencies = [ "actix-cors", "actix-http", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 62625b2f1..7faae2fdb 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "MeiliSearch HTTP server" edition = "2018" license = "MIT" name = "meilisearch-http" -version = "0.21.0-alpha.5" +version = "0.21.0-alpha.6" [[bin]] name = "meilisearch" From b1a5ef0aab2d5a389d61a2299c503d7165c43ea9 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 21:48:51 +0200 Subject: [PATCH 470/522] improve mini dashboard routing --- meilisearch-http/src/lib.rs | 111 +++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 46 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 25c29ac11..6f4ce7e43 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -15,64 +15,83 @@ pub use option::Opt; #[macro_export] macro_rules! create_app { - ($data:expr, $enable_frontend:expr) => {{ - use actix_cors::Cors; - use actix_web::middleware::TrailingSlash; - use actix_web::App; - use actix_web::{middleware, web}; - use meilisearch_http::error::payload_error_handler; - use meilisearch_http::routes::*; + ($data:expr, $enable_frontend:expr) => { + { + use actix_cors::Cors; + use actix_web::middleware::TrailingSlash; + use actix_web::{App, HttpResponse}; + use actix_web::{middleware, web}; + use meilisearch_http::error::payload_error_handler; + use meilisearch_http::routes::*; - #[cfg(feature = "mini-dashboard")] - use actix_web_static_files::ResourceFiles; + #[cfg(feature = "mini-dashboard")] + use actix_web_static_files::Resource; - #[cfg(feature = "mini-dashboard")] - mod dashboard { - include!(concat!(env!("OUT_DIR"), "/generated.rs")); - } + #[cfg(feature = "mini-dashboard")] + mod dashboard { + include!(concat!(env!("OUT_DIR"), "/generated.rs")); + } - let app = App::new() - .data($data.clone()) - .app_data( - web::JsonConfig::default() + let app = App::new() + .data($data.clone()) + .app_data( + web::JsonConfig::default() .limit($data.http_payload_size_limit()) .content_type(|_mime| true) // Accept all mime types .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .app_data( - web::QueryConfig::default() + ) + .app_data( + web::QueryConfig::default() .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .configure(document::services) - .configure(index::services) - .configure(search::services) - .configure(settings::services) - .configure(health::services) - .configure(stats::services) - .configure(key::services) - .configure(dump::services); - #[cfg(feature = "mini-dashboard")] - let app = if $enable_frontend { - let generated = dashboard::generate(); - let service = ResourceFiles::new("/", generated); - app.service(service) - } else { - app.service(running) - }; + ) + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services) + .configure(dump::services); + #[cfg(feature = "mini-dashboard")] + let app = if $enable_frontend { + let mut app = app; + let generated = dashboard::generate(); + let mut scope = web::scope("/"); + // Generate routes for mini-dashboard assets + for (path, resource) in generated.into_iter() { + let Resource {mime_type, data, ..} = resource; + // Redirect index.html to / + if path == "index.html" { + app = app.service(web::resource("/").route(web::get().to(move || { + HttpResponse::Ok().content_type(mime_type).body(data) + }))); + } else { + scope = scope.service(web::resource(path).route(web::get().to(move || { + HttpResponse::Ok().content_type(mime_type).body(data) + }))); + } + } + app.service(scope) + } else { + app.service(running) + }; - #[cfg(not(feature = "mini-dashboard"))] - let app = app.service(running); + #[cfg(not(feature = "mini-dashboard"))] + let app = app.service(running); - app.wrap( - Cors::default() + app.wrap( + Cors::default() .send_wildcard() .allowed_headers(vec!["content-type", "x-meili-api-key"]) .allow_any_origin() .allow_any_method() .max_age(86_400), // 24h - ) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - }}; + ) + .wrap(middleware::Logger::default()) + .wrap(middleware::Compress::default()) + .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) + .default_service( + web::route().to(|| HttpResponse::NotFound())) + } + }; } From 6d24a4744f5a1b0f42ba2f467522d2680d9a9bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 22 Jun 2021 20:07:23 +0200 Subject: [PATCH 471/522] Roll back facetsDistribution --- meilisearch-http/src/index/search.rs | 14 +++++++------- meilisearch-http/src/routes/search.rs | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 977992deb..507273dde 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -49,7 +49,7 @@ pub struct SearchQuery { #[serde(default = "Default::default")] pub matches: bool, pub filter: Option, - pub facet_distributions: Option>, + pub facets_distribution: Option>, } #[derive(Debug, Clone, Serialize)] @@ -73,7 +73,7 @@ pub struct SearchResult { pub offset: usize, pub processing_time_ms: u128, #[serde(skip_serializing_if = "Option::is_none")] - pub facet_distributions: Option>>, + pub facets_distribution: Option>>, } #[derive(Copy, Clone)] @@ -198,13 +198,13 @@ impl Index { let nb_hits = candidates.len(); - let facet_distributions = match query.facet_distributions { + let facets_distribution = match query.facets_distribution { Some(ref fields) => { - let mut facet_distribution = self.facets_distribution(&rtxn); + let mut facets_distribution = self.facets_distribution(&rtxn); if fields.iter().all(|f| f != "*") { - facet_distribution.facets(fields); + facets_distribution.facets(fields); } - let distribution = facet_distribution.candidates(candidates).execute()?; + let distribution = facets_distribution.candidates(candidates).execute()?; Some(distribution) } @@ -219,7 +219,7 @@ impl Index { limit: query.limit, offset: query.offset.unwrap_or_default(), processing_time_ms: before_search.elapsed().as_millis(), - facet_distributions, + facets_distribution, }; Ok(result) } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index e8f6bc86f..ab5a2f407 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -28,7 +28,7 @@ pub struct SearchQueryGet { filter: Option, #[serde(default = "Default::default")] matches: bool, - facet_distributions: Option, + facets_distribution: Option, } impl From for SearchQuery { @@ -45,8 +45,8 @@ impl From for SearchQuery { .attributes_to_highlight .map(|attrs| attrs.split(',').map(String::from).collect::>()); - let facet_distributions = other - .facet_distributions + let facets_distribution = other + .facets_distribution .map(|attrs| attrs.split(',').map(String::from).collect::>()); let filter = match other.filter { @@ -67,7 +67,7 @@ impl From for SearchQuery { attributes_to_highlight, filter, matches: other.matches, - facet_distributions, + facets_distribution, } } } From 00b0a00fc5a8d3aeb088260fb252d8d214d58a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 23 Jun 2021 11:05:30 +0200 Subject: [PATCH 472/522] Add exhaustiveFacetsCount --- meilisearch-http/src/index/search.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 507273dde..d4510a115 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -74,6 +74,8 @@ pub struct SearchResult { pub processing_time_ms: u128, #[serde(skip_serializing_if = "Option::is_none")] pub facets_distribution: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub exhaustive_facets_count: Option, } #[derive(Copy, Clone)] @@ -211,6 +213,11 @@ impl Index { None => None, }; + let exhaustive_facets_count = match facets_distribution { + Some(_) => Some(false), // not implemented yet + None => None + }; + let result = SearchResult { exhaustive_nb_hits: false, // not implemented yet hits: documents, @@ -220,6 +227,7 @@ impl Index { offset: query.offset.unwrap_or_default(), processing_time_ms: before_search.elapsed().as_millis(), facets_distribution, + exhaustive_facets_count, }; Ok(result) } From ec3140a29e3eb9fbde79a84fc19d1092edf46518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 23 Jun 2021 11:23:57 +0200 Subject: [PATCH 473/522] Fix clippy --- meilisearch-http/src/index/search.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index d4510a115..a1dc2ee55 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -213,10 +213,7 @@ impl Index { None => None, }; - let exhaustive_facets_count = match facets_distribution { - Some(_) => Some(false), // not implemented yet - None => None - }; + let exhaustive_facets_count = facets_distribution.as_ref().map(|_| false); // not implemented yet let result = SearchResult { exhaustive_nb_hits: false, // not implemented yet From da36a6b5cd266e0b7012eaaa38d2a52642f689f7 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 15:06:36 +0200 Subject: [PATCH 474/522] fix not found error --- meilisearch-http/src/index_controller/dump_actor/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/error.rs b/meilisearch-http/src/index_controller/dump_actor/error.rs index ed693d5f3..3e5e488e7 100644 --- a/meilisearch-http/src/index_controller/dump_actor/error.rs +++ b/meilisearch-http/src/index_controller/dump_actor/error.rs @@ -43,7 +43,7 @@ impl ErrorCode for DumpActorError { fn error_code(&self) -> Code { match self { DumpActorError::DumpAlreadyRunning => Code::DumpAlreadyInProgress, - DumpActorError::DumpDoesNotExist(_) => Code::DocumentNotFound, + DumpActorError::DumpDoesNotExist(_) => Code::NotFound, DumpActorError::Internal(_) => Code::Internal, DumpActorError::UuidResolver(e) => e.error_code(), DumpActorError::UpdateActor(e) => e.error_code(), From 322d6b8cfec154b5ee022b90d8a3760e25e358f3 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 15:25:56 +0200 Subject: [PATCH 475/522] fix serialization bug in settings --- meilisearch-http/src/routes/settings.rs | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/routes/settings.rs b/meilisearch-http/src/routes/settings.rs index 10ce7292e..07e9a7563 100644 --- a/meilisearch-http/src/routes/settings.rs +++ b/meilisearch-http/src/routes/settings.rs @@ -7,7 +7,7 @@ use crate::{error::ResponseError, index::Unchecked}; #[macro_export] macro_rules! make_setting_route { - ($route:literal, $type:ty, $attr:ident) => { + ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => { mod $attr { use actix_web::{web, HttpResponse}; @@ -51,7 +51,9 @@ macro_rules! make_setting_route { index_uid: actix_web::web::Path, ) -> std::result::Result { let settings = data.settings(index_uid.into_inner()).await?; - Ok(HttpResponse::Ok().json(settings.$attr)) + let mut json = serde_json::json!(&settings); + let val = json[$camelcase_attr].take(); + Ok(HttpResponse::Ok().json(val)) } } }; @@ -60,43 +62,50 @@ macro_rules! make_setting_route { make_setting_route!( "/indexes/{index_uid}/settings/filterable-attributes", std::collections::HashSet, - filterable_attributes + filterable_attributes, + "filterableAttributes" ); make_setting_route!( "/indexes/{index_uid}/settings/displayed-attributes", Vec, - displayed_attributes + displayed_attributes, + "displayedAttributes" ); make_setting_route!( "/indexes/{index_uid}/settings/searchable-attributes", Vec, - searchable_attributes + searchable_attributes, + "searchableAttributes" ); make_setting_route!( "/indexes/{index_uid}/settings/stop-words", std::collections::BTreeSet, - stop_words + stop_words, + "stopWords" ); make_setting_route!( "/indexes/{index_uid}/settings/synonyms", std::collections::BTreeMap>, - synonyms + synonyms, + "synonyms" ); make_setting_route!( "/indexes/{index_uid}/settings/distinct-attribute", String, - distinct_attribute + distinct_attribute, + "distinctAttribute" ); make_setting_route!( "/indexes/{index_uid}/settings/ranking-rules", Vec, - ranking_rules + ranking_rules, + "rankingRules" ); macro_rules! create_services { From b9b4feada8ca0fb09049b015e76fe875f2a1c840 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 16:21:32 +0200 Subject: [PATCH 476/522] add tests --- .../tests/settings/get_settings.rs | 42 +++++++++++++++++-- meilisearch-http/tests/stats/mod.rs | 7 +--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 804167517..4941b5d2f 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -1,5 +1,21 @@ +use std::collections::HashMap; + +use once_cell::sync::Lazy; +use serde_json::{Value, json}; + use crate::common::Server; -use serde_json::json; + +static DEFAULT_SETTINGS_VALUES: Lazy> = Lazy::new(|| { + let mut map = HashMap::new(); + map.insert("displayed_attributes", json!(["*"])); + map.insert("searchable_attributes", json!(["*"])); + map.insert("filterable_attributes", json!([])); + map.insert("distinct_attribute", json!(Value::Null)); + map.insert("ranking_rules", json!(["words", "typo", "proximity", "attribute", "exactness"])); + map.insert("stop_words", json!([])); + map.insert("synonyms", json!({})); + map +}); #[actix_rt::test] async fn get_settings_unexisting_index() { @@ -142,6 +158,7 @@ macro_rules! test_setting_routes { $( mod $setting { use crate::common::Server; + use super::DEFAULT_SETTINGS_VALUES; #[actix_rt::test] async fn get_unexisting_index() { @@ -177,8 +194,25 @@ macro_rules! test_setting_routes { .chars() .map(|c| if c == '_' { '-' } else { c }) .collect::()); - let (_response, code) = server.service.delete(url).await; - assert_eq!(code, 404); + let (response, code) = server.service.delete(url).await; + assert_eq!(code, 404, "{}", response); + } + + #[actix_rt::test] + async fn get_default() { + let server = Server::new().await; + let index = server.index("test"); + let (response, code) = index.create(None).await; + assert_eq!(code, 200, "{}", response); + let url = format!("/indexes/test/settings/{}", + stringify!($setting) + .chars() + .map(|c| if c == '_' { '-' } else { c }) + .collect::()); + let (response, code) = server.service.get(url).await; + assert_eq!(code, 200, "{}", response); + let expected = DEFAULT_SETTINGS_VALUES.get(stringify!($setting)).unwrap(); + assert_eq!(expected, &response); } } )* @@ -189,6 +223,8 @@ test_setting_routes!( filterable_attributes, displayed_attributes, searchable_attributes, + distinct_attribute, stop_words, + ranking_rules, synonyms ); diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index a5d08b52b..aba860256 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -54,11 +54,8 @@ async fn stats() { assert_eq!(code, 202, "{}", response); assert_eq!(response["updateId"], 0); - let (response, code) = server.stats().await; - - assert_eq!(code, 200, "{}", response); - - index.wait_update_id(0).await; + let response = index.wait_update_id(0).await; + println!("response: {}", response); let (response, code) = server.stats().await; From 71226feb74f21c99fbf8e74cf949c196e4a1ab38 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 13:21:48 +0200 Subject: [PATCH 477/522] refactor create_app macro --- meilisearch-http/src/lib.rs | 149 +++++++++++++----------- meilisearch-http/src/main.rs | 8 +- meilisearch-http/src/routes/document.rs | 3 +- 3 files changed, 83 insertions(+), 77 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 6f4ce7e43..182742a10 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -13,73 +13,83 @@ pub mod analytics; pub use self::data::Data; pub use option::Opt; +use actix_web::{HttpResponse, web}; + +pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { + let http_payload_size_limit = data.http_payload_size_limit(); + config + .data(data) + .app_data( + web::JsonConfig::default() + .limit(dbg!(http_payload_size_limit)) + .content_type(|_mime| true) // Accept all mime types + .error_handler(|err, _req| error::payload_error_handler(err).into()), + ) + .app_data(web::PayloadConfig::new(http_payload_size_limit)) + .app_data( + web::QueryConfig::default() + .error_handler(|err, _req| error::payload_error_handler(err).into()), + ); +} + +#[cfg(feature = "mini-dashboard")] +pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { + use actix_web_static_files::Resource; + + mod dashboard { + include!(concat!(env!("OUT_DIR"), "/generated.rs")); + } + + if enable_frontend { + let generated = dashboard::generate(); + let mut scope = web::scope("/"); + // Generate routes for mini-dashboard assets + for (path, resource) in generated.into_iter() { + let Resource {mime_type, data, ..} = resource; + // Redirect index.html to / + if path == "index.html" { + config.service(web::resource("/").route(web::get().to(move || { + HttpResponse::Ok().content_type(mime_type).body(data) + }))); + } else { + scope = scope.service(web::resource(path).route(web::get().to(move || { + HttpResponse::Ok().content_type(mime_type).body(data) + }))); + } + } + config.service(scope); + } else { + config.service(routes::running); + } +} + +#[cfg(not(feature = "mini-dashboard"))] +pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { + config.service(routes::running); +} + #[macro_export] macro_rules! create_app { - ($data:expr, $enable_frontend:expr) => { - { - use actix_cors::Cors; - use actix_web::middleware::TrailingSlash; - use actix_web::{App, HttpResponse}; - use actix_web::{middleware, web}; - use meilisearch_http::error::payload_error_handler; - use meilisearch_http::routes::*; + ($data:expr, $enable_frontend:expr) => {{ + use actix_cors::Cors; + use actix_web::middleware::TrailingSlash; + use actix_web::App; + use actix_web::{middleware, web}; + use meilisearch_http::routes::*; + use meilisearch_http::{dashboard, configure_data}; - #[cfg(feature = "mini-dashboard")] - use actix_web_static_files::Resource; - - #[cfg(feature = "mini-dashboard")] - mod dashboard { - include!(concat!(env!("OUT_DIR"), "/generated.rs")); - } - - let app = App::new() - .data($data.clone()) - .app_data( - web::JsonConfig::default() - .limit($data.http_payload_size_limit()) - .content_type(|_mime| true) // Accept all mime types - .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .app_data( - web::QueryConfig::default() - .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .configure(document::services) - .configure(index::services) - .configure(search::services) - .configure(settings::services) - .configure(health::services) - .configure(stats::services) - .configure(key::services) - .configure(dump::services); - #[cfg(feature = "mini-dashboard")] - let app = if $enable_frontend { - let mut app = app; - let generated = dashboard::generate(); - let mut scope = web::scope("/"); - // Generate routes for mini-dashboard assets - for (path, resource) in generated.into_iter() { - let Resource {mime_type, data, ..} = resource; - // Redirect index.html to / - if path == "index.html" { - app = app.service(web::resource("/").route(web::get().to(move || { - HttpResponse::Ok().content_type(mime_type).body(data) - }))); - } else { - scope = scope.service(web::resource(path).route(web::get().to(move || { - HttpResponse::Ok().content_type(mime_type).body(data) - }))); - } - } - app.service(scope) - } else { - app.service(running) - }; - - #[cfg(not(feature = "mini-dashboard"))] - let app = app.service(running); - - app.wrap( + App::new() + .configure(|s| configure_data(s, $data.clone())) + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services) + .configure(dump::services) + .configure(|s| dashboard(s, $enable_frontend)) + .wrap( Cors::default() .send_wildcard() .allowed_headers(vec!["content-type", "x-meili-api-key"]) @@ -87,11 +97,8 @@ macro_rules! create_app { .allow_any_method() .max_age(86_400), // 24h ) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - .default_service( - web::route().to(|| HttpResponse::NotFound())) - } - }; + .wrap(middleware::Logger::default()) + .wrap(middleware::Compress::default()) + .wrap(middleware::NormalizePath::new(middleware::TrailingSlash::Trim)) + }}; } diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 9f3e62356..c723a8ad5 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -2,7 +2,7 @@ use std::env; use actix_web::HttpServer; use main_error::MainError; -use meilisearch_http::{create_app, Data, Opt}; +use meilisearch_http::{Data, Opt, create_app}; use structopt::StructOpt; #[cfg(all(not(debug_assertions), feature = "analytics"))] @@ -74,7 +74,7 @@ async fn main() -> Result<(), MainError> { async fn run_http(data: Data, opt: Opt) -> Result<(), Box> { let _enable_dashboard = &opt.env == "development"; - let http_server = HttpServer::new(move || create_app!(&data, _enable_dashboard)) + let http_server = HttpServer::new(move || create_app!(data, _enable_dashboard)) // Disable signals allows the server to terminate immediately when a user enter CTRL-C .disable_signals(); @@ -83,8 +83,8 @@ async fn run_http(data: Data, opt: Opt) -> Result<(), Box .bind_rustls(opt.http_addr, config)? .run() .await?; - } else { - http_server.bind(opt.http_addr)?.run().await?; + } else { + http_server.bind(opt.http_addr)?.run().await?; } Ok(()) } diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 9019bdb7c..75c950734 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -1,4 +1,3 @@ -use actix_web::web::Payload; use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use indexmap::IndexMap; @@ -130,7 +129,7 @@ async fn add_documents( data: web::Data, path: web::Path, params: web::Query, - body: Payload, + body: web::Payload, ) -> Result { let update_status = data .add_documents( From 1c13100948b708909191627ca64c67630c7955c0 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 13:55:16 +0200 Subject: [PATCH 478/522] implement custom payload --- meilisearch-http/src/data/updates.rs | 3 +- meilisearch-http/src/index_controller/mod.rs | 3 +- meilisearch-http/src/lib.rs | 60 +++++++++++++++++++- meilisearch-http/src/routes/document.rs | 6 +- 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index 6b29d46b1..ea47d5a74 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,7 +1,6 @@ -use actix_web::web::Payload; use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use super::Data; +use crate::{Data, Payload}; use crate::index::{Checked, Settings}; use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus}; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index c32c40345..ddee5fd1e 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -3,7 +3,7 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; -use actix_web::web::{Bytes, Payload}; +use actix_web::web::Bytes; use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use log::error; @@ -25,6 +25,7 @@ use uuid_resolver::{error::UuidResolverError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; use error::Result; +use crate::Payload; use self::dump_actor::load_dump; use self::error::IndexControllerError; diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 182742a10..2bdb4de8e 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -10,10 +10,13 @@ pub mod routes; #[cfg(all(not(debug_assertions), feature = "analytics"))] pub mod analytics; +use std::{pin::Pin, task::{Context, Poll}}; + pub use self::data::Data; +use futures::{Stream, future::{Ready, ready}}; pub use option::Opt; -use actix_web::{HttpResponse, web}; +use actix_web::{FromRequest, HttpRequest, dev, error::PayloadError, web}; pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { let http_payload_size_limit = data.http_payload_size_limit(); @@ -35,6 +38,7 @@ pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { #[cfg(feature = "mini-dashboard")] pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { use actix_web_static_files::Resource; + use actix_web::HttpResponse; mod dashboard { include!(concat!(env!("OUT_DIR"), "/generated.rs")); @@ -102,3 +106,57 @@ macro_rules! create_app { .wrap(middleware::NormalizePath::new(middleware::TrailingSlash::Trim)) }}; } + +pub struct Payload { + payload: dev::Payload, + limit: usize, +} + +pub struct PayloadConfig { + limit: usize, +} + +impl Default for PayloadConfig { + fn default() -> Self { + Self { limit: 256 * 1024 } + } +} + +impl FromRequest for Payload { + type Config = PayloadConfig; + + type Error = PayloadError; + + type Future = Ready>; + + #[inline] + fn from_request(req: &HttpRequest, payload: &mut dev::Payload) -> Self::Future { + let limit = req.app_data::().map(|c| c.limit).unwrap_or(Self::Config::default().limit); + ready(Ok(Payload { payload: payload.take(), limit })) + } +} + +impl Stream for Payload { + type Item = Result; + + #[inline] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match Pin::new(&mut self.payload).poll_next(cx) { + Poll::Ready(Some(result)) => { + match result { + Ok(bytes) => { + match self.limit.checked_sub(bytes.len()) { + Some(new_limit) => { + self.limit = new_limit; + Poll::Ready(Some(Ok(bytes))) + } + None => Poll::Ready(Some(Err(PayloadError::Overflow))), + } + } + x => Poll::Ready(Some(x)), + } + }, + otherwise => otherwise, + } + } +} diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 75c950734..3cc483334 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -9,7 +9,7 @@ use serde_json::Value; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; -use crate::Data; +use crate::{Data, Payload}; const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; @@ -129,7 +129,7 @@ async fn add_documents( data: web::Data, path: web::Path, params: web::Query, - body: web::Payload, + body: Payload, ) -> Result { let update_status = data .add_documents( @@ -173,7 +173,7 @@ async fn update_documents( data: web::Data, path: web::Path, params: web::Query, - body: web::Payload, + body: Payload, ) -> Result { let update = data .add_documents( From 4b292c6e9b2638c8957e8ea5ba38dc3f4cdf4d9b Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 13:58:22 +0200 Subject: [PATCH 479/522] add payload limit to app config --- meilisearch-http/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 2bdb4de8e..ad3d4b86b 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -28,7 +28,7 @@ pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { .content_type(|_mime| true) // Accept all mime types .error_handler(|err, _req| error::payload_error_handler(err).into()), ) - .app_data(web::PayloadConfig::new(http_payload_size_limit)) + .app_data(PayloadConfig::new(http_payload_size_limit)) .app_data( web::QueryConfig::default() .error_handler(|err, _req| error::payload_error_handler(err).into()), @@ -116,6 +116,10 @@ pub struct PayloadConfig { limit: usize, } +impl PayloadConfig { + pub fn new(limit: usize) -> Self { Self { limit } } +} + impl Default for PayloadConfig { fn default() -> Self { Self { limit: 256 * 1024 } From f62779671bc6c4e443497b60dfcab48ff4fb3421 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 14:21:20 +0200 Subject: [PATCH 480/522] change error message for payload size limit --- .../src/index_controller/update_actor/error.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs index a95f1eafa..ae0418033 100644 --- a/meilisearch-http/src/index_controller/update_actor/error.rs +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -21,6 +21,8 @@ pub enum UpdateActorError { FatalUpdateStoreError, #[error("invalid payload: {0}")] InvalidPayload(Box), + #[error("payload error: {0}")] + PayloadError(#[from] actix_web::error::PayloadError), } impl From> for UpdateActorError { @@ -39,7 +41,6 @@ internal_error!( UpdateActorError: heed::Error, std::io::Error, serde_json::Error, - actix_http::error::PayloadError, tokio::task::JoinError ); @@ -51,6 +52,12 @@ impl ErrorCode for UpdateActorError { UpdateActorError::IndexActor(e) => e.error_code(), UpdateActorError::FatalUpdateStoreError => Code::Internal, UpdateActorError::InvalidPayload(_) => Code::BadRequest, + UpdateActorError::PayloadError(error) => { + match error { + actix_http::error::PayloadError::Overflow => Code::PayloadTooLarge, + _ => Code::Internal, + } + }, } } } From b000ae76145bfe7df033ac92be691f6cba2a4875 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 14:45:26 +0200 Subject: [PATCH 481/522] remove file if write to update file fails --- .../index_controller/update_actor/actor.rs | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 41995727a..f9b886783 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -122,7 +122,7 @@ where &self, uuid: Uuid, meta: UpdateMeta, - mut payload: mpsc::Receiver>, + payload: mpsc::Receiver>, ) -> Result { let file_path = match meta { UpdateMeta::DocumentsAddition { .. } => { @@ -137,21 +137,35 @@ where .open(&path) .await?; - let mut file_len = 0; - while let Some(bytes) = payload.recv().await { - let bytes = bytes?; - file_len += bytes.as_ref().len(); - file.write_all(bytes.as_ref()).await?; + async fn write_to_file(file: &mut fs::File, mut payload: mpsc::Receiver>) -> Result + where + D: AsRef<[u8]> + Sized + 'static, + { + let mut file_len = 0; + while let Some(bytes) = payload.recv().await { + let bytes = bytes?; + file_len += bytes.as_ref().len(); + file.write_all(bytes.as_ref()).await?; + } + Ok(file_len) } - if file_len != 0 { - file.flush().await?; - let file = file.into_std().await; - Some((file, update_file_id)) - } else { - // empty update, delete the empty file. - fs::remove_file(&path).await?; - None + let file_len = write_to_file(&mut file, payload).await; + + match file_len { + Ok(len) if len > 0 => { + file.flush().await?; + let file = file.into_std().await; + Some((file, update_file_id)) + } + Err(e) => { + fs::remove_file(&path).await?; + return Err(e) + } + _ => { + fs::remove_file(&path).await?; + None + } } } _ => None, From 834995b13030bde56a1f0fbc753fcb76f574e39e Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 14:48:33 +0200 Subject: [PATCH 482/522] clippy + fmt --- meilisearch-http/src/data/updates.rs | 2 +- meilisearch-http/src/error.rs | 9 +-- meilisearch-http/src/index/dump.rs | 2 +- meilisearch-http/src/index/mod.rs | 2 +- meilisearch-http/src/index/search.rs | 9 ++- meilisearch-http/src/index_controller/mod.rs | 2 +- .../index_controller/update_actor/actor.rs | 7 +- .../index_controller/update_actor/error.rs | 8 +-- .../update_actor/store/mod.rs | 9 ++- .../src/index_controller/updates.rs | 5 +- meilisearch-http/src/lib.rs | 68 +++++++++++-------- meilisearch-http/src/main.rs | 6 +- meilisearch-http/src/routes/search.rs | 2 +- 13 files changed, 78 insertions(+), 53 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index ea47d5a74..e729340a3 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,8 +1,8 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use crate::{Data, Payload}; use crate::index::{Checked, Settings}; use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus}; +use crate::{Data, Payload}; impl Data { pub async fn add_documents( diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 7d56de738..0b7093e80 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -103,7 +103,7 @@ impl ErrorCode for MilliError<'_> { milli::Error::UserError(ref error) => { match error { // TODO: wait for spec for new error codes. - | UserError::Csv(_) + UserError::Csv(_) | UserError::SerdeJson(_) | UserError::MaxDatabaseSizeReached | UserError::InvalidCriterionName { .. } @@ -148,9 +148,10 @@ impl ErrorCode for PayloadError { PayloadError::Json(err) => match err { JsonPayloadError::Overflow => Code::PayloadTooLarge, JsonPayloadError::ContentType => Code::UnsupportedMediaType, - JsonPayloadError::Payload(aweb::error::PayloadError::Overflow) => Code::PayloadTooLarge, - JsonPayloadError::Deserialize(_) - | JsonPayloadError::Payload(_) => Code::BadRequest, + JsonPayloadError::Payload(aweb::error::PayloadError::Overflow) => { + Code::PayloadTooLarge + } + JsonPayloadError::Deserialize(_) | JsonPayloadError::Payload(_) => Code::BadRequest, JsonPayloadError::Serialize(_) => Code::Internal, _ => Code::Internal, }, diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 52a5cabd9..263e3bd52 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -3,7 +3,7 @@ use std::io::{BufRead, BufReader, Write}; use std::path::Path; use std::sync::Arc; -use anyhow::{Context, bail}; +use anyhow::{bail, Context}; use heed::RoTxn; use indexmap::IndexMap; use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 5a005fdc0..7227f8d35 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -13,7 +13,7 @@ use serde_json::{Map, Value}; use crate::helpers::EnvSizer; use error::Result; -pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT, default_crop_length}; +pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use updates::{Checked, Facets, Settings, Unchecked}; use self::error::IndexError; diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index a1dc2ee55..bc6d2ae74 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -233,8 +233,8 @@ impl Index { fn compute_matches>( matcher: &impl Matcher, document: &Document, - analyzer: &Analyzer - ) -> MatchesInfo { + analyzer: &Analyzer, +) -> MatchesInfo { let mut matches = BTreeMap::new(); for (key, value) in document { @@ -1174,6 +1174,9 @@ mod test { let analyzer = Analyzer::new(config); let matches = compute_matches(&matcher, &value, &analyzer); - assert_eq!(format!("{:?}", matches), r##"{"about": [MatchInfo { start: 0, length: 6 }, MatchInfo { start: 31, length: 7 }, MatchInfo { start: 191, length: 7 }, MatchInfo { start: 225, length: 7 }, MatchInfo { start: 233, length: 6 }], "color": [MatchInfo { start: 0, length: 3 }]}"##); + assert_eq!( + format!("{:?}", matches), + r##"{"about": [MatchInfo { start: 0, length: 6 }, MatchInfo { start: 31, length: 7 }, MatchInfo { start: 191, length: 7 }, MatchInfo { start: 225, length: 7 }, MatchInfo { start: 233, length: 6 }], "color": [MatchInfo { start: 0, length: 3 }]}"## + ); } } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index ddee5fd1e..b7dba1a8a 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -24,8 +24,8 @@ use uuid_resolver::{error::UuidResolverError, UuidResolverHandle}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; -use error::Result; use crate::Payload; +use error::Result; use self::dump_actor::load_dump; use self::error::IndexControllerError; diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index f9b886783..5b269ea9a 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -137,7 +137,10 @@ where .open(&path) .await?; - async fn write_to_file(file: &mut fs::File, mut payload: mpsc::Receiver>) -> Result + async fn write_to_file( + file: &mut fs::File, + mut payload: mpsc::Receiver>, + ) -> Result where D: AsRef<[u8]> + Sized + 'static, { @@ -160,7 +163,7 @@ where } Err(e) => { fs::remove_file(&path).await?; - return Err(e) + return Err(e); } _ => { fs::remove_file(&path).await?; diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs index ae0418033..6f0e848c3 100644 --- a/meilisearch-http/src/index_controller/update_actor/error.rs +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -52,11 +52,9 @@ impl ErrorCode for UpdateActorError { UpdateActorError::IndexActor(e) => e.error_code(), UpdateActorError::FatalUpdateStoreError => Code::Internal, UpdateActorError::InvalidPayload(_) => Code::BadRequest, - UpdateActorError::PayloadError(error) => { - match error { - actix_http::error::PayloadError::Overflow => Code::PayloadTooLarge, - _ => Code::Internal, - } + UpdateActorError::PayloadError(error) => match error { + actix_http::error::PayloadError::Overflow => Code::PayloadTooLarge, + _ => Code::Internal, }, } } diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 1ee30c99d..21a320b26 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -572,7 +572,10 @@ fn update_uuid_to_file_path(root: impl AsRef, uuid: Uuid) -> PathBuf { #[cfg(test)] mod test { use super::*; - use crate::index_controller::{UpdateResult, index_actor::{MockIndexActorHandle, error::IndexActorError}}; + use crate::index_controller::{ + index_actor::{error::IndexActorError, MockIndexActorHandle}, + UpdateResult, + }; use futures::future::ok; @@ -651,7 +654,9 @@ mod test { if processing.id() == 0 { Box::pin(ok(Ok(processing.process(UpdateResult::Other)))) } else { - Box::pin(ok(Err(processing.fail(IndexActorError::ExistingPrimaryKey.into())))) + Box::pin(ok(Err( + processing.fail(IndexActorError::ExistingPrimaryKey.into()) + ))) } }); diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index 13ad9f7e5..d02438d3c 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -3,7 +3,10 @@ use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::{error::ResponseError, index::{Settings, Unchecked}}; +use crate::{ + error::ResponseError, + index::{Settings, Unchecked}, +}; #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index ad3d4b86b..0ca09234c 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -10,13 +10,19 @@ pub mod routes; #[cfg(all(not(debug_assertions), feature = "analytics"))] pub mod analytics; -use std::{pin::Pin, task::{Context, Poll}}; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; pub use self::data::Data; -use futures::{Stream, future::{Ready, ready}}; +use futures::{ + future::{ready, Ready}, + Stream, +}; pub use option::Opt; -use actix_web::{FromRequest, HttpRequest, dev, error::PayloadError, web}; +use actix_web::{dev, error::PayloadError, web, FromRequest, HttpRequest}; pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { let http_payload_size_limit = data.http_payload_size_limit(); @@ -40,12 +46,12 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { use actix_web_static_files::Resource; use actix_web::HttpResponse; - mod dashboard { + mod generated { include!(concat!(env!("OUT_DIR"), "/generated.rs")); } if enable_frontend { - let generated = dashboard::generate(); + let generated = generated::generate(); let mut scope = web::scope("/"); // Generate routes for mini-dashboard assets for (path, resource) in generated.into_iter() { @@ -80,7 +86,7 @@ macro_rules! create_app { use actix_web::App; use actix_web::{middleware, web}; use meilisearch_http::routes::*; - use meilisearch_http::{dashboard, configure_data}; + use meilisearch_http::{configure_data, dashboard}; App::new() .configure(|s| configure_data(s, $data.clone())) @@ -95,15 +101,17 @@ macro_rules! create_app { .configure(|s| dashboard(s, $enable_frontend)) .wrap( Cors::default() - .send_wildcard() - .allowed_headers(vec!["content-type", "x-meili-api-key"]) - .allow_any_origin() - .allow_any_method() - .max_age(86_400), // 24h + .send_wildcard() + .allowed_headers(vec!["content-type", "x-meili-api-key"]) + .allow_any_origin() + .allow_any_method() + .max_age(86_400), // 24h ) .wrap(middleware::Logger::default()) .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(middleware::TrailingSlash::Trim)) + .wrap(middleware::NormalizePath::new( + middleware::TrailingSlash::Trim, + )) }}; } @@ -117,12 +125,14 @@ pub struct PayloadConfig { } impl PayloadConfig { - pub fn new(limit: usize) -> Self { Self { limit } } + pub fn new(limit: usize) -> Self { + Self { limit } + } } impl Default for PayloadConfig { fn default() -> Self { - Self { limit: 256 * 1024 } + Self { limit: 256 * 1024 } } } @@ -135,8 +145,14 @@ impl FromRequest for Payload { #[inline] fn from_request(req: &HttpRequest, payload: &mut dev::Payload) -> Self::Future { - let limit = req.app_data::().map(|c| c.limit).unwrap_or(Self::Config::default().limit); - ready(Ok(Payload { payload: payload.take(), limit })) + let limit = req + .app_data::() + .map(|c| c.limit) + .unwrap_or(Self::Config::default().limit); + ready(Ok(Payload { + payload: payload.take(), + limit, + })) } } @@ -146,19 +162,15 @@ impl Stream for Payload { #[inline] fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match Pin::new(&mut self.payload).poll_next(cx) { - Poll::Ready(Some(result)) => { - match result { - Ok(bytes) => { - match self.limit.checked_sub(bytes.len()) { - Some(new_limit) => { - self.limit = new_limit; - Poll::Ready(Some(Ok(bytes))) - } - None => Poll::Ready(Some(Err(PayloadError::Overflow))), - } + Poll::Ready(Some(result)) => match result { + Ok(bytes) => match self.limit.checked_sub(bytes.len()) { + Some(new_limit) => { + self.limit = new_limit; + Poll::Ready(Some(Ok(bytes))) } - x => Poll::Ready(Some(x)), - } + None => Poll::Ready(Some(Err(PayloadError::Overflow))), + }, + x => Poll::Ready(Some(x)), }, otherwise => otherwise, } diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index c723a8ad5..50201d912 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -2,7 +2,7 @@ use std::env; use actix_web::HttpServer; use main_error::MainError; -use meilisearch_http::{Data, Opt, create_app}; +use meilisearch_http::{create_app, Data, Opt}; use structopt::StructOpt; #[cfg(all(not(debug_assertions), feature = "analytics"))] @@ -83,8 +83,8 @@ async fn run_http(data: Data, opt: Opt) -> Result<(), Box .bind_rustls(opt.http_addr, config)? .run() .await?; - } else { - http_server.bind(opt.http_addr)?.run().await?; + } else { + http_server.bind(opt.http_addr)?.run().await?; } Ok(()) } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index ab5a2f407..f596f7e4c 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -6,7 +6,7 @@ use serde_json::Value; use crate::error::ResponseError; use crate::helpers::Authentication; -use crate::index::{SearchQuery, default_crop_length, DEFAULT_SEARCH_LIMIT}; +use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; From a838238a639c7ac1646d0f3656075df376b7de4d Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 14:56:02 +0200 Subject: [PATCH 483/522] move payload to own module --- meilisearch-http/src/data/updates.rs | 3 +- meilisearch-http/src/extractors/mod.rs | 1 + meilisearch-http/src/extractors/payload.rs | 69 ++++++++++++++++++ meilisearch-http/src/index_controller/mod.rs | 2 +- meilisearch-http/src/lib.rs | 76 ++------------------ meilisearch-http/src/routes/document.rs | 3 +- 6 files changed, 79 insertions(+), 75 deletions(-) create mode 100644 meilisearch-http/src/extractors/mod.rs create mode 100644 meilisearch-http/src/extractors/payload.rs diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index e729340a3..4e38294e9 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,8 +1,9 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use crate::extractors::payload::Payload; use crate::index::{Checked, Settings}; use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus}; -use crate::{Data, Payload}; +use crate::Data; impl Data { pub async fn add_documents( diff --git a/meilisearch-http/src/extractors/mod.rs b/meilisearch-http/src/extractors/mod.rs new file mode 100644 index 000000000..fbb091fe2 --- /dev/null +++ b/meilisearch-http/src/extractors/mod.rs @@ -0,0 +1 @@ +pub mod payload; diff --git a/meilisearch-http/src/extractors/payload.rs b/meilisearch-http/src/extractors/payload.rs new file mode 100644 index 000000000..260561e40 --- /dev/null +++ b/meilisearch-http/src/extractors/payload.rs @@ -0,0 +1,69 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +use actix_http::error::PayloadError; +use actix_web::{dev, web, FromRequest, HttpRequest}; +use futures::future::{ready, Ready}; +use futures::Stream; + +pub struct Payload { + payload: dev::Payload, + limit: usize, +} + +pub struct PayloadConfig { + limit: usize, +} + +impl PayloadConfig { + pub fn new(limit: usize) -> Self { + Self { limit } + } +} + +impl Default for PayloadConfig { + fn default() -> Self { + Self { limit: 256 * 1024 } + } +} + +impl FromRequest for Payload { + type Config = PayloadConfig; + + type Error = PayloadError; + + type Future = Ready>; + + #[inline] + fn from_request(req: &HttpRequest, payload: &mut dev::Payload) -> Self::Future { + let limit = req + .app_data::() + .map(|c| c.limit) + .unwrap_or(Self::Config::default().limit); + ready(Ok(Payload { + payload: payload.take(), + limit, + })) + } +} + +impl Stream for Payload { + type Item = Result; + + #[inline] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match Pin::new(&mut self.payload).poll_next(cx) { + Poll::Ready(Some(result)) => match result { + Ok(bytes) => match self.limit.checked_sub(bytes.len()) { + Some(new_limit) => { + self.limit = new_limit; + Poll::Ready(Some(Ok(bytes))) + } + None => Poll::Ready(Some(Err(PayloadError::Overflow))), + }, + x => Poll::Ready(Some(x)), + }, + otherwise => otherwise, + } + } +} diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index b7dba1a8a..c223ead76 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -22,9 +22,9 @@ use update_actor::UpdateActorHandle; pub use updates::*; use uuid_resolver::{error::UuidResolverError, UuidResolverHandle}; +use crate::extractors::payload::Payload; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; use crate::option::Opt; -use crate::Payload; use error::Result; use self::dump_actor::load_dump; diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 0ca09234c..de30e1782 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,6 +1,7 @@ pub mod data; #[macro_use] pub mod error; +pub mod extractors; pub mod helpers; mod index; mod index_controller; @@ -10,19 +11,12 @@ pub mod routes; #[cfg(all(not(debug_assertions), feature = "analytics"))] pub mod analytics; -use std::{ - pin::Pin, - task::{Context, Poll}, -}; - pub use self::data::Data; -use futures::{ - future::{ready, Ready}, - Stream, -}; pub use option::Opt; -use actix_web::{dev, error::PayloadError, web, FromRequest, HttpRequest}; +use actix_web::web; + +use extractors::payload::PayloadConfig; pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { let http_payload_size_limit = data.http_payload_size_limit(); @@ -114,65 +108,3 @@ macro_rules! create_app { )) }}; } - -pub struct Payload { - payload: dev::Payload, - limit: usize, -} - -pub struct PayloadConfig { - limit: usize, -} - -impl PayloadConfig { - pub fn new(limit: usize) -> Self { - Self { limit } - } -} - -impl Default for PayloadConfig { - fn default() -> Self { - Self { limit: 256 * 1024 } - } -} - -impl FromRequest for Payload { - type Config = PayloadConfig; - - type Error = PayloadError; - - type Future = Ready>; - - #[inline] - fn from_request(req: &HttpRequest, payload: &mut dev::Payload) -> Self::Future { - let limit = req - .app_data::() - .map(|c| c.limit) - .unwrap_or(Self::Config::default().limit); - ready(Ok(Payload { - payload: payload.take(), - limit, - })) - } -} - -impl Stream for Payload { - type Item = Result; - - #[inline] - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match Pin::new(&mut self.payload).poll_next(cx) { - Poll::Ready(Some(result)) => match result { - Ok(bytes) => match self.limit.checked_sub(bytes.len()) { - Some(new_limit) => { - self.limit = new_limit; - Poll::Ready(Some(Ok(bytes))) - } - None => Poll::Ready(Some(Err(PayloadError::Overflow))), - }, - x => Poll::Ready(Some(x)), - }, - otherwise => otherwise, - } - } -} diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 3cc483334..a9d6c0db8 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -7,9 +7,10 @@ use serde::Deserialize; use serde_json::Value; use crate::error::ResponseError; +use crate::extractors::payload::Payload; use crate::helpers::Authentication; use crate::routes::IndexParam; -use crate::{Data, Payload}; +use crate::Data; const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; From 880fc069bd786bc00bf922dfdb98c0301f2c339f Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 15:02:00 +0200 Subject: [PATCH 484/522] remove dbg --- meilisearch-http/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index de30e1782..46fea718c 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -24,7 +24,7 @@ pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { .data(data) .app_data( web::JsonConfig::default() - .limit(dbg!(http_payload_size_limit)) + .limit(http_payload_size_limit) .content_type(|_mime| true) // Accept all mime types .error_handler(|err, _req| error::payload_error_handler(err).into()), ) From f68c2574521c5e7fec1e12c5587a7b5f2476c526 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 16:34:07 +0200 Subject: [PATCH 485/522] move flush in write_to_file function --- meilisearch-http/src/index_controller/update_actor/actor.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 5b269ea9a..5bc2c727e 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -145,11 +145,15 @@ where D: AsRef<[u8]> + Sized + 'static, { let mut file_len = 0; + while let Some(bytes) = payload.recv().await { let bytes = bytes?; file_len += bytes.as_ref().len(); file.write_all(bytes.as_ref()).await?; } + + file.flush().await?; + Ok(file_len) } @@ -157,7 +161,6 @@ where match file_len { Ok(len) if len > 0 => { - file.flush().await?; let file = file.into_std().await; Some((file, update_file_id)) } From 74bb748a4e0c83561e159c8b253a57c4354f5ad4 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 18:40:19 +0200 Subject: [PATCH 486/522] bump milli to 0.6.0 --- Cargo.lock | 4 ++-- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/error.rs | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b50e8f84..1eee0fe65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1723,8 +1723,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.5.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.5.0#b073fd49ea04fbb8da940f6357c952b34af94c0e" +version = "0.6.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.6.0#c38b0b883d602a7efe9ba87f36157f4fc35f6ec7" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 7faae2fdb..5d2fdbcad 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.5.0" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.6.0" } mime = "0.3.16" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 7d56de738..2d2d0d87b 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -119,6 +119,7 @@ impl ErrorCode for MilliError<'_> { UserError::PrimaryKeyCannotBeChanged => Code::PrimaryKeyAlreadyPresent, UserError::PrimaryKeyCannotBeReset => Code::PrimaryKeyAlreadyPresent, UserError::UnknownInternalDocumentId { .. } => Code::DocumentNotFound, + UserError::InvalidFacetsDistribution { .. } => Code::BadRequest, } } } From a1d34faaad9442cd116e31c8a7f60fc5dbdf6035 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 10:53:51 +0200 Subject: [PATCH 487/522] decompose error messages --- meilisearch-http/src/error.rs | 4 ++-- meilisearch-http/src/index/error.rs | 8 ++++---- .../src/index_controller/dump_actor/error.rs | 10 +++++----- meilisearch-http/src/index_controller/error.rs | 14 +++++++------- .../src/index_controller/index_actor/error.rs | 10 +++++----- .../src/index_controller/update_actor/error.rs | 10 +++++----- .../src/index_controller/uuid_resolver/error.rs | 8 ++++---- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 695b2603a..6124ed880 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -12,9 +12,9 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, thiserror::Error)] pub enum AuthenticationError { - #[error("you must have an authorization token")] + #[error("You must have an authorization token")] MissingAuthorizationHeader, - #[error("invalid API key")] + #[error("Invalid API key")] InvalidToken(String), } diff --git a/meilisearch-http/src/index/error.rs b/meilisearch-http/src/index/error.rs index b9bf71a3b..cfae11a1f 100644 --- a/meilisearch-http/src/index/error.rs +++ b/meilisearch-http/src/index/error.rs @@ -9,11 +9,11 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum IndexError { - #[error("internal error: {0}")] + #[error("Internal error: {0}")] Internal(Box), - #[error("document with id {0} not found.")] + #[error("Document with id {0} not found.")] DocumentNotFound(String), - #[error("error with facet: {0}")] + #[error("{0}")] Facet(#[from] FacetError), #[error("{0}")] Milli(#[from] milli::Error), @@ -39,7 +39,7 @@ impl ErrorCode for IndexError { #[derive(Debug, thiserror::Error)] pub enum FacetError { - #[error("invalid facet expression, expected {}, found: {1}", .0.join(", "))] + #[error("Invalid facet expression, expected {}, found: {1}", .0.join(", "))] InvalidExpression(&'static [&'static str], Value), } diff --git a/meilisearch-http/src/index_controller/dump_actor/error.rs b/meilisearch-http/src/index_controller/dump_actor/error.rs index 3e5e488e7..b6bddb5ea 100644 --- a/meilisearch-http/src/index_controller/dump_actor/error.rs +++ b/meilisearch-http/src/index_controller/dump_actor/error.rs @@ -7,15 +7,15 @@ pub type Result = std::result::Result; #[derive(thiserror::Error, Debug)] pub enum DumpActorError { - #[error("dump already running")] + #[error("Another dump is already in progress")] DumpAlreadyRunning, - #[error("dump `{0}` does not exist")] + #[error("Dump `{0}` not found")] DumpDoesNotExist(String), - #[error("internal error: {0}")] + #[error("Internal error: {0}")] Internal(Box), - #[error("error while dumping uuids: {0}")] + #[error("{0}")] UuidResolver(#[from] UuidResolverError), - #[error("error while dumping updates: {0}")] + #[error("{0}")] UpdateActor(#[from] UpdateActorError), } diff --git a/meilisearch-http/src/index_controller/error.rs b/meilisearch-http/src/index_controller/error.rs index c01eb24c0..00f6b8656 100644 --- a/meilisearch-http/src/index_controller/error.rs +++ b/meilisearch-http/src/index_controller/error.rs @@ -12,24 +12,24 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum IndexControllerError { - #[error("missing index uid")] + #[error("Index creation must have an uid")] MissingUid, - #[error("index resolution error: {0}")] + #[error("{0}")] Uuid(#[from] UuidResolverError), - #[error("error with index: {0}")] + #[error("{0}")] IndexActor(#[from] IndexActorError), - #[error("error with update: {0}")] + #[error("{0}")] UpdateActor(#[from] UpdateActorError), - #[error("error with dump: {0}")] + #[error("{0}")] DumpActor(#[from] DumpActorError), - #[error("error with index: {0}")] + #[error("{0}")] IndexError(#[from] IndexError), } impl ErrorCode for IndexControllerError { fn error_code(&self) -> Code { match self { - IndexControllerError::MissingUid => Code::InvalidIndexUid, + IndexControllerError::MissingUid => Code::BadRequest, IndexControllerError::Uuid(e) => e.error_code(), IndexControllerError::IndexActor(e) => e.error_code(), IndexControllerError::UpdateActor(e) => e.error_code(), diff --git a/meilisearch-http/src/index_controller/index_actor/error.rs b/meilisearch-http/src/index_controller/index_actor/error.rs index 244797234..12a81796b 100644 --- a/meilisearch-http/src/index_controller/index_actor/error.rs +++ b/meilisearch-http/src/index_controller/index_actor/error.rs @@ -6,15 +6,15 @@ pub type Result = std::result::Result; #[derive(thiserror::Error, Debug)] pub enum IndexActorError { - #[error("index error: {0}")] + #[error("{0}")] IndexError(#[from] IndexError), - #[error("index already exists")] + #[error("Index already exists")] IndexAlreadyExists, - #[error("index doesn't exists")] + #[error("Index not found")] UnexistingIndex, - #[error("existing primary key")] + #[error("A primary key is already present. It's impossible to update it")] ExistingPrimaryKey, - #[error("internal Index Error: {0}")] + #[error("Internal Error: {0}")] Internal(Box), #[error("{0}")] Milli(#[from] milli::Error), diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs index 6f0e848c3..29c1802a8 100644 --- a/meilisearch-http/src/index_controller/update_actor/error.rs +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -9,19 +9,19 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] #[allow(clippy::large_enum_variant)] pub enum UpdateActorError { - #[error("update {0} doesn't exist.")] + #[error("Update {0} not found.")] UnexistingUpdate(u64), - #[error("internal error processing update: {0}")] + #[error("Internal error: {0}")] Internal(Box), - #[error("error with index: {0}")] + #[error("{0}")] IndexActor(#[from] IndexActorError), #[error( "update store was shut down due to a fatal error, please check your logs for more info." )] FatalUpdateStoreError, - #[error("invalid payload: {0}")] + #[error("{0}")] InvalidPayload(Box), - #[error("payload error: {0}")] + #[error("{0}")] PayloadError(#[from] actix_web::error::PayloadError), } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/error.rs b/meilisearch-http/src/index_controller/uuid_resolver/error.rs index 3d7fb8444..de3dc662e 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/error.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/error.rs @@ -4,13 +4,13 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum UuidResolverError { - #[error("name already exist.")] + #[error("Index already exists.")] NameAlreadyExist, - #[error("index \"{0}\" doesn't exist.")] + #[error("Index \"{0}\" not found.")] UnexistingIndex(String), - #[error("badly formatted index uid: {0}")] + #[error("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")] BadlyFormatted(String), - #[error("internal error resolving index uid: {0}")] + #[error("Internal error: {0}")] Internal(Box), } From b4fd4212ad9109ec42d22e72b8795e03b23b6250 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 23 Jun 2021 10:41:55 +0200 Subject: [PATCH 488/522] reduce the log level of some info! --- meilisearch-http/src/index/updates.rs | 8 ++++---- meilisearch-http/src/index_controller/dump_actor/actor.rs | 6 +++--- meilisearch-http/src/index_controller/dump_actor/mod.rs | 4 ++-- meilisearch-http/src/index_controller/snapshot.rs | 8 ++++---- .../src/index_controller/update_actor/actor.rs | 4 ++-- .../src/index_controller/uuid_resolver/actor.rs | 4 ++-- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 6ace7e375..09535721f 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -4,7 +4,7 @@ use std::marker::PhantomData; use std::num::NonZeroUsize; use flate2::read::GzDecoder; -use log::info; +use log::{debug, info, trace}; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize, Serializer}; @@ -201,7 +201,7 @@ impl Index { update_builder: UpdateBuilder, primary_key: Option<&str>, ) -> Result { - info!("performing document addition"); + trace!("performing document addition"); // Set the primary key if not set already, ignore if already set. if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) { @@ -215,7 +215,7 @@ impl Index { builder.index_documents_method(method); let indexing_callback = - |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); + |indexing_step, update_id| debug!("update {}: {:?}", update_id, indexing_step); let gzipped = false; let addition = match content { @@ -300,7 +300,7 @@ impl Index { } builder.execute(|indexing_step, update_id| { - info!("update {}: {:?}", update_id, indexing_step) + debug!("update {}: {:?}", update_id, indexing_step) })?; Ok(UpdateResult::Other) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 4924df947..eee733c4a 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use async_stream::stream; use chrono::Utc; use futures::{lock::Mutex, stream::StreamExt}; -use log::{error, info}; +use log::{error, trace}; use tokio::sync::{mpsc, oneshot, RwLock}; use update_actor::UpdateActorHandle; use uuid_resolver::UuidResolverHandle; @@ -60,7 +60,7 @@ where } pub async fn run(mut self) { - info!("Started dump actor."); + trace!("Started dump actor."); let mut inbox = self .inbox @@ -135,7 +135,7 @@ where match task_result { Ok(Ok(())) => { dump_infos.done(); - info!("Dump succeed"); + trace!("Dump succeed"); } Ok(Err(e)) => { dump_infos.with_error(e.to_string()); diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 057b8ebaf..a73740b02 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use chrono::{DateTime, Utc}; -use log::{info, warn}; +use log::{info, trace, warn}; #[cfg(test)] use mockall::automock; use serde::{Deserialize, Serialize}; @@ -164,7 +164,7 @@ where P: UpdateActorHandle + Send + Sync + Clone + 'static, { async fn run(self) -> Result<()> { - info!("Performing dump."); + trace!("Performing dump."); create_dir_all(&self.path).await?; diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs index 9f0c5c0ba..7bdedae76 100644 --- a/meilisearch-http/src/index_controller/snapshot.rs +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::bail; -use log::{error, info}; +use log::{error, info, trace}; use tokio::fs; use tokio::task::spawn_blocking; use tokio::time::sleep; @@ -47,14 +47,14 @@ where ); loop { if let Err(e) = self.perform_snapshot().await { - error!("{}", e); + error!("Error while performing snapshot: {}", e); } sleep(self.snapshot_period).await; } } async fn perform_snapshot(&self) -> anyhow::Result<()> { - info!("Performing snapshot."); + trace!("Performing snapshot."); let snapshot_dir = self.snapshot_path.clone(); fs::create_dir_all(&snapshot_dir).await?; @@ -87,7 +87,7 @@ where }) .await??; - info!("Created snapshot in {:?}.", snapshot_path); + trace!("Created snapshot in {:?}.", snapshot_path); Ok(()) } diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 5bc2c727e..51e8cb28c 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_stream::stream; use futures::StreamExt; -use log::info; +use log::{trace}; use oxidized_json_checker::JsonChecker; use tokio::fs; use tokio::io::AsyncWriteExt; @@ -61,7 +61,7 @@ where pub async fn run(mut self) { use UpdateMsg::*; - info!("Started update actor."); + trace!("Started update actor."); let mut inbox = self .inbox diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index efad15c92..d221bd4f2 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -1,6 +1,6 @@ use std::{collections::HashSet, path::PathBuf}; -use log::{info, warn}; +use log::{trace, warn}; use tokio::sync::mpsc; use uuid::Uuid; @@ -19,7 +19,7 @@ impl UuidResolverActor { pub async fn run(mut self) { use UuidResolveMsg::*; - info!("uuid resolver started"); + trace!("uuid resolver started"); loop { match self.inbox.recv().await { From 36f32f58d4bd030678ebc18cc3869257b9ac3f48 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 23 Jun 2021 11:07:19 +0200 Subject: [PATCH 489/522] add the log_level variable to the cli and reduce the log level of milli and grenad --- meilisearch-http/src/main.rs | 8 ++++++-- meilisearch-http/src/option.rs | 4 ++++ meilisearch-http/tests/common/server.rs | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 50201d912..3c51b7e72 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -19,8 +19,12 @@ const SENTRY_DSN: &str = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/306 async fn main() -> Result<(), MainError> { let opt = Opt::from_args(); - let mut log_builder = - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")); + let mut log_builder = env_logger::Builder::new(); + log_builder.parse_filters(&opt.log_level); + if opt.log_level == "info" { + // if we are in info we only allow the warn log_level for milli + log_builder.filter_module("milli", log::LevelFilter::Warn); + } match opt.env.as_ref() { "production" => { diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 8cf2e0aba..0e75b63c8 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -188,6 +188,10 @@ pub struct Opt { #[structopt(long, conflicts_with = "import-snapshot")] pub import_dump: Option, + /// Set the log level + #[structopt(long, env = "MEILI_LOG_LEVEL", default_value = "info")] + pub log_level: String, + #[structopt(skip)] pub indexer_options: IndexerOpts, } diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index cd9dfb2d0..6cf1acb6a 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -91,5 +91,6 @@ pub fn default_settings(dir: impl AsRef) -> Opt { snapshot_interval_sec: 0, import_dump: None, indexer_options: IndexerOpts::default(), + log_level: "off".into(), } } From ad8d9a97d6cfcb30988a51e25cacc94bc79216e8 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 23 Jun 2021 12:18:34 +0200 Subject: [PATCH 490/522] debug the body of every http request --- meilisearch-http/src/index/search.rs | 4 ++-- meilisearch-http/src/index_controller/mod.rs | 4 ++-- meilisearch-http/src/routes/document.rs | 17 ++++++++++++++--- meilisearch-http/src/routes/dump.rs | 3 +++ meilisearch-http/src/routes/index.rs | 9 +++++++++ meilisearch-http/src/routes/search.rs | 5 +++++ meilisearch-http/src/routes/settings.rs | 8 ++++++++ meilisearch-http/src/routes/stats.rs | 3 +++ 8 files changed, 46 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index bc6d2ae74..2d8095559 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -33,7 +33,7 @@ pub const fn default_crop_length() -> usize { DEFAULT_CROP_LENGTH } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchQuery { pub q: Option, @@ -62,7 +62,7 @@ pub struct SearchHit { pub matches_info: Option, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] #[serde(rename_all = "camelCase")] pub struct SearchResult { pub hits: Vec, diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index c223ead76..a90498b9c 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -55,7 +55,7 @@ pub struct IndexSettings { pub primary_key: Option, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] #[serde(rename_all = "camelCase")] pub struct IndexStats { #[serde(skip)] @@ -76,7 +76,7 @@ pub struct IndexController { dump_handle: dump_actor::DumpActorHandleImpl, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] #[serde(rename_all = "camelCase")] pub struct Stats { pub database_size: u64, diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index a9d6c0db8..6ac521f79 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -1,7 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use indexmap::IndexMap; -use log::error; +use log::{debug, error}; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; use serde_json::Value; @@ -64,6 +64,7 @@ async fn get_document( let document = data .retrieve_document(index, id, None as Option>) .await?; + debug!("returns: {:?}", document); Ok(HttpResponse::Ok().json(document)) } @@ -78,10 +79,11 @@ async fn delete_document( let update_status = data .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]) .await?; + debug!("returns: {:?}", update_status); Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct BrowseQuery { offset: Option, @@ -95,6 +97,7 @@ async fn get_all_documents( path: web::Path, params: web::Query, ) -> Result { + debug!("called with params: {:?}", params); let attributes_to_retrieve = params.attributes_to_retrieve.as_ref().and_then(|attrs| { let mut names = Vec::new(); for name in attrs.split(',').map(String::from) { @@ -114,10 +117,11 @@ async fn get_all_documents( attributes_to_retrieve, ) .await?; + debug!("returns: {:?}", documents); Ok(HttpResponse::Ok().json(documents)) } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateDocumentsQuery { primary_key: Option, @@ -132,6 +136,7 @@ async fn add_documents( params: web::Query, body: Payload, ) -> Result { + debug!("called with params: {:?}", params); let update_status = data .add_documents( path.into_inner().index_uid, @@ -142,6 +147,7 @@ async fn add_documents( ) .await?; + debug!("returns: {:?}", update_status); Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } @@ -176,6 +182,7 @@ async fn update_documents( params: web::Query, body: Payload, ) -> Result { + debug!("called with params: {:?}", params); let update = data .add_documents( path.into_inner().index_uid, @@ -186,6 +193,7 @@ async fn update_documents( ) .await?; + debug!("returns: {:?}", update); Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() }))) } @@ -198,6 +206,7 @@ async fn delete_documents( path: web::Path, body: web::Json>, ) -> Result { + debug!("called with params: {:?}", body); let ids = body .iter() .map(|v| { @@ -208,6 +217,7 @@ async fn delete_documents( .collect(); let update_status = data.delete_documents(path.index_uid.clone(), ids).await?; + debug!("returns: {:?}", update_status); Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } @@ -218,5 +228,6 @@ async fn clear_all_documents( path: web::Path, ) -> Result { let update_status = data.clear_documents(path.index_uid.clone()).await?; + debug!("returns: {:?}", update_status); Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 370eef509..f905207ec 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -1,5 +1,6 @@ use actix_web::HttpResponse; use actix_web::{get, post, web}; +use log::debug; use serde::{Deserialize, Serialize}; use crate::error::ResponseError; @@ -14,6 +15,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { async fn create_dump(data: web::Data) -> Result { let res = data.create_dump().await?; + debug!("returns: {:?}", res); Ok(HttpResponse::Accepted().json(res)) } @@ -35,5 +37,6 @@ async fn get_dump_status( ) -> Result { let res = data.dump_status(path.dump_uid.clone()).await?; + debug!("returns: {:?}", res); Ok(HttpResponse::Ok().json(res)) } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 137b4dbbd..0f29c985e 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -1,6 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; +use log::debug; use serde::{Deserialize, Serialize}; use super::{IndexParam, UpdateStatusResponse}; @@ -21,6 +22,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { #[get("/indexes", wrap = "Authentication::Private")] async fn list_indexes(data: web::Data) -> Result { let indexes = data.list_indexes().await?; + debug!("returns: {:?}", indexes); Ok(HttpResponse::Ok().json(indexes)) } @@ -30,6 +32,7 @@ async fn get_index( path: web::Path, ) -> Result { let meta = data.index(path.index_uid.clone()).await?; + debug!("returns: {:?}", meta); Ok(HttpResponse::Ok().json(meta)) } @@ -45,8 +48,10 @@ async fn create_index( data: web::Data, body: web::Json, ) -> Result { + debug!("called with params: {:?}", body); let body = body.into_inner(); let meta = data.create_index(body.uid, body.primary_key).await?; + debug!("returns: {:?}", meta); Ok(HttpResponse::Ok().json(meta)) } @@ -73,10 +78,12 @@ async fn update_index( path: web::Path, body: web::Json, ) -> Result { + debug!("called with params: {:?}", body); let body = body.into_inner(); let meta = data .update_index(path.into_inner().index_uid, body.primary_key, body.uid) .await?; + debug!("returns: {:?}", meta); Ok(HttpResponse::Ok().json(meta)) } @@ -108,6 +115,7 @@ async fn get_update_status( .get_update_status(params.index_uid, params.update_id) .await?; let meta = UpdateStatusResponse::from(meta); + debug!("returns: {:?}", meta); Ok(HttpResponse::Ok().json(meta)) } @@ -122,5 +130,6 @@ async fn get_all_updates_status( .map(UpdateStatusResponse::from) .collect::>(); + debug!("returns: {:?}", metas); Ok(HttpResponse::Ok().json(metas)) } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index f596f7e4c..cc1306e71 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeSet, HashSet}; use actix_web::{get, post, web, HttpResponse}; +use log::debug; use serde::Deserialize; use serde_json::Value; @@ -78,8 +79,10 @@ async fn search_with_url_query( path: web::Path, params: web::Query, ) -> Result { + debug!("called with params: {:?}", params); let query = params.into_inner().into(); let search_result = data.search(path.into_inner().index_uid, query).await?; + debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } @@ -89,8 +92,10 @@ async fn search_with_post( path: web::Path, params: web::Json, ) -> Result { + debug!("search called with params: {:?}", params); let search_result = data .search(path.into_inner().index_uid, params.into_inner()) .await?; + debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } diff --git a/meilisearch-http/src/routes/settings.rs b/meilisearch-http/src/routes/settings.rs index 07e9a7563..5e4ecf1a4 100644 --- a/meilisearch-http/src/routes/settings.rs +++ b/meilisearch-http/src/routes/settings.rs @@ -1,4 +1,5 @@ use actix_web::{delete, get, post, web, HttpResponse}; +use log::debug; use crate::helpers::Authentication; use crate::index::Settings; @@ -9,6 +10,7 @@ use crate::{error::ResponseError, index::Unchecked}; macro_rules! make_setting_route { ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => { mod $attr { + use log::debug; use actix_web::{web, HttpResponse}; use crate::data; @@ -27,6 +29,7 @@ macro_rules! make_setting_route { ..Default::default() }; let update_status = data.update_settings(index_uid.into_inner(), settings, false).await?; + debug!("returns: {:?}", update_status); Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } @@ -42,6 +45,7 @@ macro_rules! make_setting_route { }; let update_status = data.update_settings(index_uid.into_inner(), settings, true).await?; + debug!("returns: {:?}", update_status); Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } @@ -51,6 +55,7 @@ macro_rules! make_setting_route { index_uid: actix_web::web::Path, ) -> std::result::Result { let settings = data.settings(index_uid.into_inner()).await?; + debug!("returns: {:?}", settings); let mut json = serde_json::json!(&settings); let val = json[$camelcase_attr].take(); Ok(HttpResponse::Ok().json(val)) @@ -145,6 +150,7 @@ async fn update_all( .update_settings(index_uid.into_inner(), settings, true) .await?; let json = serde_json::json!({ "updateId": update_result.id() }); + debug!("returns: {:?}", json); Ok(HttpResponse::Accepted().json(json)) } @@ -154,6 +160,7 @@ async fn get_all( index_uid: web::Path, ) -> Result { let settings = data.settings(index_uid.into_inner()).await?; + debug!("returns: {:?}", settings); Ok(HttpResponse::Ok().json(settings)) } @@ -167,5 +174,6 @@ async fn delete_all( .update_settings(index_uid.into_inner(), settings, false) .await?; let json = serde_json::json!({ "updateId": update_result.id() }); + debug!("returns: {:?}", json); Ok(HttpResponse::Accepted().json(json)) } diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index f2d1ddecc..c39989188 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,6 +1,7 @@ use actix_web::get; use actix_web::web; use actix_web::HttpResponse; +use log::debug; use serde::Serialize; use crate::error::ResponseError; @@ -21,6 +22,7 @@ async fn get_index_stats( ) -> Result { let response = data.get_index_stats(path.index_uid.clone()).await?; + debug!("returns: {:?}", response); Ok(HttpResponse::Ok().json(response)) } @@ -28,6 +30,7 @@ async fn get_index_stats( async fn get_stats(data: web::Data) -> Result { let response = data.get_all_stats().await?; + debug!("returns: {:?}", response); Ok(HttpResponse::Ok().json(response)) } From 5229f1e220b27eef72a4084c76e1dc502729ccc4 Mon Sep 17 00:00:00 2001 From: marin postma Date: Tue, 22 Jun 2021 23:49:34 +0200 Subject: [PATCH 491/522] experimental auth extractor --- meilisearch-http/src/routes/search.rs | 127 ++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index cc1306e71..2fbbfe4b3 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -1,18 +1,129 @@ -use std::collections::{BTreeSet, HashSet}; +use std::any::{Any, TypeId}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::marker::PhantomData; +use std::ops::Deref; -use actix_web::{get, post, web, HttpResponse}; use log::debug; +use actix_web::{web, FromRequest, HttpResponse}; +use futures::future::{err, ok, Ready}; use serde::Deserialize; use serde_json::Value; -use crate::error::ResponseError; -use crate::helpers::Authentication; +use crate::error::{AuthenticationError, ResponseError}; use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; +struct Public; + +impl Policy for Public { + fn authenticate(&self, _token: &[u8]) -> bool { + true + } +} + +struct GuardedData { + data: D, + _marker: PhantomData, +} + +trait Policy { + fn authenticate(&self, token: &[u8]) -> bool; +} + +struct Policies { + inner: HashMap>, +} + +impl Policies { + fn new() -> Self { + Self { inner: HashMap::new() } + } + + fn insert(&mut self, policy: S) { + self.inner.insert(TypeId::of::(), Box::new(policy)); + } + + fn get(&self) -> Option<&S> { + self.inner + .get(&TypeId::of::()) + .and_then(|p| p.downcast_ref::()) + } +} + +enum AuthConfig { + NoAuth, + Auth(Policies), +} + +impl Default for AuthConfig { + fn default() -> Self { + Self::NoAuth + } +} + +impl FromRequest for GuardedData { + type Config = AuthConfig; + + type Error = ResponseError; + + type Future = Ready>; + + fn from_request( + req: &actix_web::HttpRequest, + _payload: &mut actix_http::Payload, + ) -> Self::Future { + match req.app_data::() { + Some(config) => match config { + AuthConfig::NoAuth => match req.app_data::().cloned() { + Some(data) => ok(Self { + data, + _marker: PhantomData, + }), + None => todo!("Data not configured"), + }, + AuthConfig::Auth(policies) => match policies.get::

() { + Some(policy) => match req.headers().get("x-meili-api-key") { + Some(token) => { + if policy.authenticate(token.as_bytes()) { + match req.app_data::().cloned() { + Some(data) => ok(Self { + data, + _marker: PhantomData, + }), + None => todo!("Data not configured"), + } + } else { + err(AuthenticationError::InvalidToken(String::from("hello")).into()) + } + } + None => err(AuthenticationError::MissingAuthorizationHeader.into()), + }, + None => todo!("no policy found"), + }, + }, + None => todo!(), + } + } +} + +impl Deref for GuardedData { + type Target = D; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(search_with_post).service(search_with_url_query); + let mut policies = Policies::new(); + policies.insert(Public); + cfg.service( + web::resource("/indexes/{index_uid}/search") + .app_data(AuthConfig::Auth(policies)) + .route(web::get().to(search_with_url_query)) + .route(web::post().to(search_with_post)), + ); } #[derive(Deserialize, Debug)] @@ -73,9 +184,8 @@ impl From for SearchQuery { } } -#[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")] async fn search_with_url_query( - data: web::Data, + data: GuardedData, path: web::Path, params: web::Query, ) -> Result { @@ -86,9 +196,8 @@ async fn search_with_url_query( Ok(HttpResponse::Ok().json(search_result)) } -#[post("/indexes/{index_uid}/search", wrap = "Authentication::Public")] async fn search_with_post( - data: web::Data, + data: GuardedData, path: web::Path, params: web::Json, ) -> Result { From 12f6709e1c61c2960c8f5d8d6e2813017be5af2f Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 18:54:33 +0200 Subject: [PATCH 492/522] move authencation to extractor mod --- .../src/extractors/authentication/mod.rs | 117 ++++++++++++++++++ meilisearch-http/src/extractors/mod.rs | 1 + meilisearch-http/src/routes/search.rs | 112 +---------------- 3 files changed, 122 insertions(+), 108 deletions(-) create mode 100644 meilisearch-http/src/extractors/authentication/mod.rs diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs new file mode 100644 index 000000000..a31847945 --- /dev/null +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -0,0 +1,117 @@ +use std::collections::HashMap; +use std::marker::PhantomData; +use std::ops::Deref; +use std::any::{Any, TypeId}; + +use actix_web::FromRequest; +use futures::future::err; +use futures::future::{Ready, ok}; + +use crate::error::{AuthenticationError, ResponseError}; + +pub struct Public; + +impl Policy for Public { + fn authenticate(&self, _token: &[u8]) -> bool { + true + } +} + +pub struct GuardedData { + data: D, + _marker: PhantomData, +} + +impl Deref for GuardedData { + type Target = D; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +pub trait Policy { + fn authenticate(&self, token: &[u8]) -> bool; +} + +pub struct Policies { + inner: HashMap>, +} + +impl Policies { + pub fn new() -> Self { + Self { inner: HashMap::new() } + } + + pub fn insert(&mut self, policy: S) { + self.inner.insert(TypeId::of::(), Box::new(policy)); + } + + pub fn get(&self) -> Option<&S> { + self.inner + .get(&TypeId::of::()) + .and_then(|p| p.downcast_ref::()) + } +} + +impl Default for Policies { + fn default() -> Self { + Self::new() + } +} + +pub enum AuthConfig { + NoAuth, + Auth(Policies), +} + +impl Default for AuthConfig { + fn default() -> Self { + Self::NoAuth + } +} + +impl FromRequest for GuardedData { + type Config = AuthConfig; + + type Error = ResponseError; + + type Future = Ready>; + + fn from_request( + req: &actix_web::HttpRequest, + _payload: &mut actix_http::Payload, + ) -> Self::Future { + match req.app_data::() { + Some(config) => match config { + AuthConfig::NoAuth => match req.app_data::().cloned() { + Some(data) => ok(Self { + data, + _marker: PhantomData, + }), + None => todo!("Data not configured"), + }, + AuthConfig::Auth(policies) => match policies.get::

() { + Some(policy) => match req.headers().get("x-meili-api-key") { + Some(token) => { + if policy.authenticate(token.as_bytes()) { + match req.app_data::().cloned() { + Some(data) => ok(Self { + data, + _marker: PhantomData, + }), + None => todo!("Data not configured"), + } + } else { + err(AuthenticationError::InvalidToken(String::from("hello")).into()) + } + } + None => err(AuthenticationError::MissingAuthorizationHeader.into()), + }, + None => todo!("no policy found"), + }, + }, + None => todo!(), + } + } +} diff --git a/meilisearch-http/src/extractors/mod.rs b/meilisearch-http/src/extractors/mod.rs index fbb091fe2..8d2942f1d 100644 --- a/meilisearch-http/src/extractors/mod.rs +++ b/meilisearch-http/src/extractors/mod.rs @@ -1 +1,2 @@ pub mod payload; +pub mod authentication; diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 2fbbfe4b3..0d3184ca9 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -1,119 +1,15 @@ -use std::any::{Any, TypeId}; -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::marker::PhantomData; -use std::ops::Deref; +use std::collections::{BTreeSet, HashSet}; use log::debug; -use actix_web::{web, FromRequest, HttpResponse}; -use futures::future::{err, ok, Ready}; +use actix_web::{web, HttpResponse}; use serde::Deserialize; use serde_json::Value; -use crate::error::{AuthenticationError, ResponseError}; +use crate::error::ResponseError; use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; - -struct Public; - -impl Policy for Public { - fn authenticate(&self, _token: &[u8]) -> bool { - true - } -} - -struct GuardedData { - data: D, - _marker: PhantomData, -} - -trait Policy { - fn authenticate(&self, token: &[u8]) -> bool; -} - -struct Policies { - inner: HashMap>, -} - -impl Policies { - fn new() -> Self { - Self { inner: HashMap::new() } - } - - fn insert(&mut self, policy: S) { - self.inner.insert(TypeId::of::(), Box::new(policy)); - } - - fn get(&self) -> Option<&S> { - self.inner - .get(&TypeId::of::()) - .and_then(|p| p.downcast_ref::()) - } -} - -enum AuthConfig { - NoAuth, - Auth(Policies), -} - -impl Default for AuthConfig { - fn default() -> Self { - Self::NoAuth - } -} - -impl FromRequest for GuardedData { - type Config = AuthConfig; - - type Error = ResponseError; - - type Future = Ready>; - - fn from_request( - req: &actix_web::HttpRequest, - _payload: &mut actix_http::Payload, - ) -> Self::Future { - match req.app_data::() { - Some(config) => match config { - AuthConfig::NoAuth => match req.app_data::().cloned() { - Some(data) => ok(Self { - data, - _marker: PhantomData, - }), - None => todo!("Data not configured"), - }, - AuthConfig::Auth(policies) => match policies.get::

() { - Some(policy) => match req.headers().get("x-meili-api-key") { - Some(token) => { - if policy.authenticate(token.as_bytes()) { - match req.app_data::().cloned() { - Some(data) => ok(Self { - data, - _marker: PhantomData, - }), - None => todo!("Data not configured"), - } - } else { - err(AuthenticationError::InvalidToken(String::from("hello")).into()) - } - } - None => err(AuthenticationError::MissingAuthorizationHeader.into()), - }, - None => todo!("no policy found"), - }, - }, - None => todo!(), - } - } -} - -impl Deref for GuardedData { - type Target = D; - - fn deref(&self) -> &Self::Target { - &self.data - } -} +use crate::extractors::authentication::{Policies, AuthConfig, Public, GuardedData}; pub fn services(cfg: &mut web::ServiceConfig) { let mut policies = Policies::new(); From 5b717513916f8b201af37b960b1b50f6a13567b4 Mon Sep 17 00:00:00 2001 From: marin postma Date: Wed, 23 Jun 2021 19:35:26 +0200 Subject: [PATCH 493/522] policies macros --- .../src/extractors/authentication/mod.rs | 64 +++++++++++++++++-- meilisearch-http/src/extractors/mod.rs | 1 + meilisearch-http/src/lib.rs | 4 ++ meilisearch-http/src/routes/search.rs | 2 +- 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index a31847945..315461578 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::ops::Deref; use std::any::{Any, TypeId}; @@ -9,12 +9,59 @@ use futures::future::{Ready, ok}; use crate::error::{AuthenticationError, ResponseError}; -pub struct Public; +macro_rules! create_policies { + ($($name:ident), *) => { + $( + pub struct $name { + inner: HashSet> + } -impl Policy for Public { - fn authenticate(&self, _token: &[u8]) -> bool { - true - } + impl $name { + pub fn new() -> Self { + Self { inner: HashSet::new() } + } + + pub fn add(&mut self, token: Vec) { + self.inner.insert(token); + } + } + + impl Policy for $name { + fn authenticate(&self, token: &[u8]) -> bool { + self.inner.contains(token) + } + } + )* + }; +} + +create_policies!(Public, Private, Admin); + +/// Instanciate a `Policies`, filled with the given policies. +macro_rules! init_policies { + ($($name:ident), *) => { + { + let mut policies = Policies::new(); + $( + let policy = $name::new(); + policies.insert(policy); + )* + policies + } + }; +} + +/// Adds user to all specified policies. +macro_rules! create_users { + ($policies:ident, $($user:literal => { $($policy:ty), * }), *) => { + { + $( + $( + $policies.get_mut::<$policy>().map(|p| p.add($user.to_owned())) + )* + )* + } + }; } pub struct GuardedData { @@ -52,6 +99,11 @@ impl Policies { .get(&TypeId::of::()) .and_then(|p| p.downcast_ref::()) } + + pub fn get_mut(&mut self) -> Option<&mut S> { + self.inner.get_mut(&TypeId::of::()) + .and_then(|p| p.downcast_mut::()) + } } impl Default for Policies { diff --git a/meilisearch-http/src/extractors/mod.rs b/meilisearch-http/src/extractors/mod.rs index 8d2942f1d..09a56e4a0 100644 --- a/meilisearch-http/src/extractors/mod.rs +++ b/meilisearch-http/src/extractors/mod.rs @@ -1,2 +1,3 @@ pub mod payload; +#[macro_use] pub mod authentication; diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 46fea718c..0ee1fb4c2 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -35,6 +35,10 @@ pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { ); } +pub fn configure_auth(config: &mut web::ServiceConfig, opt: &Options) { + todo!() +} + #[cfg(feature = "mini-dashboard")] pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { use actix_web_static_files::Resource; diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 0d3184ca9..160660daf 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -13,7 +13,7 @@ use crate::extractors::authentication::{Policies, AuthConfig, Public, GuardedDat pub fn services(cfg: &mut web::ServiceConfig) { let mut policies = Policies::new(); - policies.insert(Public); + policies.insert(Public::new()); cfg.service( web::resource("/indexes/{index_uid}/search") .app_data(AuthConfig::Auth(policies)) From 0c1c7a3dd9085fb2d7a4708b19a582fa59f82d3b Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 14:22:12 +0200 Subject: [PATCH 494/522] implement authentication policies --- .../src/extractors/authentication/mod.rs | 58 +++++++++++-------- meilisearch-http/src/lib.rs | 30 ++++++++-- 2 files changed, 60 insertions(+), 28 deletions(-) diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index 315461578..2cce6911d 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -1,37 +1,43 @@ -use std::collections::{HashMap, HashSet}; +use std::any::{Any, TypeId}; +use std::collections::HashMap; use std::marker::PhantomData; use std::ops::Deref; -use std::any::{Any, TypeId}; use actix_web::FromRequest; use futures::future::err; -use futures::future::{Ready, ok}; +use futures::future::{ok, Ready}; use crate::error::{AuthenticationError, ResponseError}; macro_rules! create_policies { ($($name:ident), *) => { - $( - pub struct $name { - inner: HashSet> - } + pub mod policies { + use std::collections::HashSet; + use crate::extractors::authentication::Policy; - impl $name { - pub fn new() -> Self { - Self { inner: HashSet::new() } + $( + #[derive(Debug)] + pub struct $name { + inner: HashSet> } - pub fn add(&mut self, token: Vec) { - self.inner.insert(token); - } - } + impl $name { + pub fn new() -> Self { + Self { inner: HashSet::new() } + } - impl Policy for $name { - fn authenticate(&self, token: &[u8]) -> bool { - self.inner.contains(token) + pub fn add(&mut self, token: Vec) { + &mut self.inner.insert(token); + } } - } - )* + + impl Policy for $name { + fn authenticate(&self, token: &[u8]) -> bool { + self.inner.contains(token) + } + } + )* + } }; } @@ -41,7 +47,7 @@ create_policies!(Public, Private, Admin); macro_rules! init_policies { ($($name:ident), *) => { { - let mut policies = Policies::new(); + let mut policies = crate::extractors::authentication::Policies::new(); $( let policy = $name::new(); policies.insert(policy); @@ -53,11 +59,11 @@ macro_rules! init_policies { /// Adds user to all specified policies. macro_rules! create_users { - ($policies:ident, $($user:literal => { $($policy:ty), * }), *) => { + ($policies:ident, $($user:expr => { $($policy:ty), * }), *) => { { $( $( - $policies.get_mut::<$policy>().map(|p| p.add($user.to_owned())) + $policies.get_mut::<$policy>().map(|p| p.add($user.to_owned())); )* )* } @@ -81,13 +87,16 @@ pub trait Policy { fn authenticate(&self, token: &[u8]) -> bool; } +#[derive(Debug)] pub struct Policies { inner: HashMap>, } impl Policies { pub fn new() -> Self { - Self { inner: HashMap::new() } + Self { + inner: HashMap::new(), + } } pub fn insert(&mut self, policy: S) { @@ -101,7 +110,8 @@ impl Policies { } pub fn get_mut(&mut self) -> Option<&mut S> { - self.inner.get_mut(&TypeId::of::()) + self.inner + .get_mut(&TypeId::of::()) .and_then(|p| p.downcast_mut::()) } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 0ee1fb4c2..7b9dbdc21 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,6 +1,7 @@ pub mod data; #[macro_use] pub mod error; +#[macro_use] pub mod extractors; pub mod helpers; mod index; @@ -11,17 +12,21 @@ pub mod routes; #[cfg(all(not(debug_assertions), feature = "analytics"))] pub mod analytics; +use crate::extractors::authentication::AuthConfig; + pub use self::data::Data; pub use option::Opt; use actix_web::web; use extractors::payload::PayloadConfig; +use extractors::authentication::policies::*; pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { let http_payload_size_limit = data.http_payload_size_limit(); config - .data(data) + .data(data.clone()) + .app_data(data) .app_data( web::JsonConfig::default() .limit(http_payload_size_limit) @@ -35,8 +40,24 @@ pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { ); } -pub fn configure_auth(config: &mut web::ServiceConfig, opt: &Options) { - todo!() +pub fn configure_auth(config: &mut web::ServiceConfig, data: &Data) { + let keys = data.api_keys(); + let auth_config = if let Some(ref master_key) = keys.master { + let private_key = keys.private.as_ref().unwrap(); + let public_key = keys.public.as_ref().unwrap(); + let mut policies = init_policies!(Public, Private, Admin); + create_users!( + policies, + master_key.as_bytes() => { Admin, Private, Public }, + private_key.as_bytes() => { Private, Public }, + public_key.as_bytes() => { Public } + ); + AuthConfig::Auth(policies) + } else { + AuthConfig::NoAuth + }; + + config.app_data(auth_config); } #[cfg(feature = "mini-dashboard")] @@ -84,10 +105,11 @@ macro_rules! create_app { use actix_web::App; use actix_web::{middleware, web}; use meilisearch_http::routes::*; - use meilisearch_http::{configure_data, dashboard}; + use meilisearch_http::{configure_data, dashboard, configure_auth}; App::new() .configure(|s| configure_data(s, $data.clone())) + .configure(|s| configure_auth(s, &$data)) .configure(document::services) .configure(index::services) .configure(search::services) From adf91d286b99ba25935b00a68097d9efb3c90802 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:02:35 +0200 Subject: [PATCH 495/522] update documents and search routes --- meilisearch-http/src/routes/document.rs | 87 +++++++------------------ meilisearch-http/src/routes/search.rs | 7 +- 2 files changed, 27 insertions(+), 67 deletions(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 6ac521f79..4824ec7b8 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -1,14 +1,12 @@ -use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; -use indexmap::IndexMap; -use log::{debug, error}; +use log::debug; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; use serde_json::Value; use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::payload::Payload; -use crate::helpers::Authentication; use crate::routes::IndexParam; use crate::Data; @@ -17,7 +15,6 @@ const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; macro_rules! guard_content_type { ($fn_name:ident, $guard_value:literal) => { - #[allow(dead_code)] fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { if let Some(content_type) = head.headers.get("Content-Type") { content_type @@ -33,8 +30,6 @@ macro_rules! guard_content_type { guard_content_type!(guard_json, "application/json"); -type Document = IndexMap; - #[derive(Deserialize)] struct DocumentParam { index_uid: String, @@ -42,21 +37,26 @@ struct DocumentParam { } pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get_document) - .service(delete_document) - .service(get_all_documents) - .service(add_documents) - .service(update_documents) - .service(delete_documents) - .service(clear_all_documents); + cfg.service( + web::resource("/indexes/{index_uid}/documents") + .route(web::get().to(get_all_documents)) + .route(web::post().guard(guard_json).to(add_documents)) + .route(web::put().guard(guard_json).to(update_documents)) + .route(web::delete().to(clear_all_documents)), + ) + .service( + web::scope("/indexes/{index_uid}/documents/") + .service( + web::resource("{document_id}") + .route(web::get().to(get_document)) + .route(web::delete().to(delete_document)), + ) + .route("/delete-batch", web::post().to(delete_documents)), + ); } -#[get( - "/indexes/{index_uid}/documents/{document_id}", - wrap = "Authentication::Public" -)] async fn get_document( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { let index = path.index_uid.clone(); @@ -68,12 +68,8 @@ async fn get_document( Ok(HttpResponse::Ok().json(document)) } -#[delete( - "/indexes/{index_uid}/documents/{document_id}", - wrap = "Authentication::Private" -)] async fn delete_document( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { let update_status = data @@ -91,9 +87,8 @@ struct BrowseQuery { attributes_to_retrieve: Option, } -#[get("/indexes/{index_uid}/documents", wrap = "Authentication::Public")] async fn get_all_documents( - data: web::Data, + data: GuardedData, path: web::Path, params: web::Query, ) -> Result { @@ -129,9 +124,8 @@ struct UpdateDocumentsQuery { /// Route used when the payload type is "application/json" /// Used to add or replace documents -#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn add_documents( - data: web::Data, + data: GuardedData, path: web::Path, params: web::Query, body: Payload, @@ -151,33 +145,8 @@ async fn add_documents( Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } -/// Default route for adding documents, this should return an error and redirect to the documentation -#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] -async fn add_documents_default( - _data: web::Data, - _path: web::Path, - _params: web::Query, - _body: web::Json>, -) -> Result { - error!("Unknown document type"); - todo!() -} - -/// Default route for adding documents, this should return an error and redirect to the documentation -#[put("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] -async fn update_documents_default( - _data: web::Data, - _path: web::Path, - _params: web::Query, - _body: web::Json>, -) -> Result { - error!("Unknown document type"); - todo!() -} - -#[put("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn update_documents( - data: web::Data, + data: GuardedData, path: web::Path, params: web::Query, body: Payload, @@ -197,12 +166,8 @@ async fn update_documents( Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() }))) } -#[post( - "/indexes/{index_uid}/documents/delete-batch", - wrap = "Authentication::Private" -)] async fn delete_documents( - data: web::Data, + data: GuardedData, path: web::Path, body: web::Json>, ) -> Result { @@ -221,10 +186,8 @@ async fn delete_documents( Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } -/// delete all documents -#[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn clear_all_documents( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { let update_status = data.clear_documents(path.index_uid.clone()).await?; diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 160660daf..7f1e265f6 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -9,14 +9,11 @@ use crate::error::ResponseError; use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; -use crate::extractors::authentication::{Policies, AuthConfig, Public, GuardedData}; +use crate::extractors::authentication::{GuardedData, policies::*}; pub fn services(cfg: &mut web::ServiceConfig) { - let mut policies = Policies::new(); - policies.insert(Public::new()); cfg.service( web::resource("/indexes/{index_uid}/search") - .app_data(AuthConfig::Auth(policies)) .route(web::get().to(search_with_url_query)) .route(web::post().to(search_with_post)), ); @@ -81,7 +78,7 @@ impl From for SearchQuery { } async fn search_with_url_query( - data: GuardedData, + data: GuardedData, path: web::Path, params: web::Query, ) -> Result { From ce4fb8ce20a9a4fbcdc76ab48a32b798c31ef458 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:06:58 +0200 Subject: [PATCH 496/522] update dump route --- meilisearch-http/src/routes/dump.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index f905207ec..f2f276332 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -1,18 +1,17 @@ -use actix_web::HttpResponse; -use actix_web::{get, post, web}; use log::debug; +use actix_web::{web, HttpResponse}; use serde::{Deserialize, Serialize}; use crate::error::ResponseError; -use crate::helpers::Authentication; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(create_dump).service(get_dump_status); + cfg.route("/dumps", web::post().to(create_dump)) + .route("/dumps/{dump_uid}/status", web::get().to(get_dump_status)); } -#[post("/dumps", wrap = "Authentication::Private")] -async fn create_dump(data: web::Data) -> Result { +async fn create_dump(data: GuardedData) -> Result { let res = data.create_dump().await?; debug!("returns: {:?}", res); @@ -30,9 +29,8 @@ struct DumpParam { dump_uid: String, } -#[get("/dumps/{dump_uid}/status", wrap = "Authentication::Private")] async fn get_dump_status( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { let res = data.dump_status(path.dump_uid.clone()).await?; From b044608b25b9d35cd1ca1f1b4be98168cb84bc2b Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:08:45 +0200 Subject: [PATCH 497/522] update health route --- meilisearch-http/src/routes/health.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/meilisearch-http/src/routes/health.rs b/meilisearch-http/src/routes/health.rs index 8994df722..172298861 100644 --- a/meilisearch-http/src/routes/health.rs +++ b/meilisearch-http/src/routes/health.rs @@ -1,13 +1,11 @@ -use actix_web::get; use actix_web::{web, HttpResponse}; use crate::error::ResponseError; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get_health); + cfg.route("/healts", web::get().to(get_health)); } -#[get("/health")] async fn get_health() -> Result { Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) } From fab50256bc97b571b8b5fbf8620a62cd72100693 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:33:21 +0200 Subject: [PATCH 498/522] update index routes --- meilisearch-http/src/routes/index.rs | 97 ++++++++++++++-------------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 0f29c985e..76a6d5da8 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -1,4 +1,3 @@ -use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; use log::debug; @@ -6,34 +5,29 @@ use serde::{Deserialize, Serialize}; use super::{IndexParam, UpdateStatusResponse}; use crate::error::ResponseError; -use crate::helpers::Authentication; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(list_indexes) - .service(get_index) - .service(create_index) - .service(update_index) - .service(delete_index) - .service(get_update_status) - .service(get_all_updates_status); -} - -#[get("/indexes", wrap = "Authentication::Private")] -async fn list_indexes(data: web::Data) -> Result { - let indexes = data.list_indexes().await?; - debug!("returns: {:?}", indexes); - Ok(HttpResponse::Ok().json(indexes)) -} - -#[get("/indexes/{index_uid}", wrap = "Authentication::Private")] -async fn get_index( - data: web::Data, - path: web::Path, -) -> Result { - let meta = data.index(path.index_uid.clone()).await?; - debug!("returns: {:?}", meta); - Ok(HttpResponse::Ok().json(meta)) + cfg.service( + web::resource("indexes") + .route(web::get().to(list_indexes)) + .route(web::post().to(create_index)), + ) + .service( + web::resource("/indexes/{index_uid}") + .route(web::get().to(get_index)) + .route(web::put().to(update_index)) + .route(web::delete().to(delete_index)), + ) + .route( + "/indexes/{index_uid}/updates", + web::get().to(get_all_updates_status), + ) + .route( + "/indexes/{index_uid}/updates/{update_id}", + web::get().to(get_update_status), + ); } #[derive(Debug, Deserialize)] @@ -43,18 +37,6 @@ struct IndexCreateRequest { primary_key: Option, } -#[post("/indexes", wrap = "Authentication::Private")] -async fn create_index( - data: web::Data, - body: web::Json, -) -> Result { - debug!("called with params: {:?}", body); - let body = body.into_inner(); - let meta = data.create_index(body.uid, body.primary_key).await?; - debug!("returns: {:?}", meta); - Ok(HttpResponse::Ok().json(meta)) -} - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateIndexRequest { @@ -72,9 +54,32 @@ pub struct UpdateIndexResponse { primary_key: Option, } -#[put("/indexes/{index_uid}", wrap = "Authentication::Private")] +async fn list_indexes(data: GuardedData) -> Result { + let indexes = data.list_indexes().await?; + debug!("returns: {:?}", indexes); + Ok(HttpResponse::Ok().json(indexes)) +} + +async fn create_index( + data: GuardedData, + body: web::Json, +) -> Result { + let body = body.into_inner(); + let meta = data.create_index(body.uid, body.primary_key).await?; + Ok(HttpResponse::Ok().json(meta)) +} + +async fn get_index( + data: GuardedData, + path: web::Path, +) -> Result { + let meta = data.index(path.index_uid.clone()).await?; + debug!("returns: {:?}", meta); + Ok(HttpResponse::Ok().json(meta)) +} + async fn update_index( - data: web::Data, + data: GuardedData, path: web::Path, body: web::Json, ) -> Result { @@ -87,9 +92,8 @@ async fn update_index( Ok(HttpResponse::Ok().json(meta)) } -#[delete("/indexes/{index_uid}", wrap = "Authentication::Private")] async fn delete_index( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { data.delete_index(path.index_uid.clone()).await?; @@ -102,12 +106,8 @@ struct UpdateParam { update_id: u64, } -#[get( - "/indexes/{index_uid}/updates/{update_id}", - wrap = "Authentication::Private" -)] async fn get_update_status( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { let params = path.into_inner(); @@ -119,9 +119,8 @@ async fn get_update_status( Ok(HttpResponse::Ok().json(meta)) } -#[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] async fn get_all_updates_status( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { let metas = data.get_updates_status(path.into_inner().index_uid).await?; From 817fcfdd8881ac503d3a10bf318f93acc5524db2 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:37:18 +0200 Subject: [PATCH 499/522] update keys route --- meilisearch-http/src/routes/key.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/routes/key.rs b/meilisearch-http/src/routes/key.rs index b44d747c8..51b1c78a2 100644 --- a/meilisearch-http/src/routes/key.rs +++ b/meilisearch-http/src/routes/key.rs @@ -1,13 +1,11 @@ -use actix_web::get; -use actix_web::web; -use actix_web::HttpResponse; +use actix_web::{web, HttpResponse}; use serde::Serialize; -use crate::helpers::Authentication; use crate::Data; +use crate::extractors::authentication::{GuardedData, policies::*}; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(list); + cfg.route("/keys", web::get().to(list)); } #[derive(Serialize)] @@ -16,8 +14,7 @@ struct KeysResponse { public: Option, } -#[get("/keys", wrap = "Authentication::Admin")] -async fn list(data: web::Data) -> HttpResponse { +async fn list(data: GuardedData) -> HttpResponse { let api_keys = data.api_keys.clone(); HttpResponse::Ok().json(&KeysResponse { private: api_keys.private, From 1e9f374ff8d2b25f6c6904ddb937f6be91c5499d Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:40:04 +0200 Subject: [PATCH 500/522] update running route --- meilisearch-http/src/lib.rs | 4 ++-- meilisearch-http/src/routes/mod.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 7b9dbdc21..78d9ec8b1 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -88,13 +88,13 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { } config.service(scope); } else { - config.service(routes::running); + config.route("/", web::get().to(routes::running)); } } #[cfg(not(feature = "mini-dashboard"))] pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { - config.service(routes::running); + config.route("/", web::get().to(routes::running)); } #[macro_export] diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index f4581ebcb..520949cd8 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use actix_web::{get, HttpResponse}; +use actix_web::HttpResponse; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -220,7 +220,6 @@ impl IndexUpdateResponse { /// "status": "Meilisearch is running" /// } /// ``` -#[get("/")] pub async fn running() -> HttpResponse { HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" })) } From 549b489c8ad300836aa2e40792f9522b4631d104 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:50:46 +0200 Subject: [PATCH 501/522] update settings routes --- meilisearch-http/src/routes/settings.rs | 52 ++++++++++++------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/meilisearch-http/src/routes/settings.rs b/meilisearch-http/src/routes/settings.rs index 5e4ecf1a4..f7df7b701 100644 --- a/meilisearch-http/src/routes/settings.rs +++ b/meilisearch-http/src/routes/settings.rs @@ -1,7 +1,7 @@ -use actix_web::{delete, get, post, web, HttpResponse}; use log::debug; +use actix_web::{web, HttpResponse}; -use crate::helpers::Authentication; +use crate::extractors::authentication::{GuardedData, policies::*}; use crate::index::Settings; use crate::Data; use crate::{error::ResponseError, index::Unchecked}; @@ -11,16 +11,15 @@ macro_rules! make_setting_route { ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => { mod $attr { use log::debug; - use actix_web::{web, HttpResponse}; + use actix_web::{web, HttpResponse, Resource}; use crate::data; use crate::error::ResponseError; - use crate::helpers::Authentication; use crate::index::Settings; + use crate::extractors::authentication::{GuardedData, policies::*}; - #[actix_web::delete($route, wrap = "Authentication::Private")] - pub async fn delete( - data: web::Data, + async fn delete( + data: GuardedData, index_uid: web::Path, ) -> Result { use crate::index::Settings; @@ -33,9 +32,8 @@ macro_rules! make_setting_route { Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } - #[actix_web::post($route, wrap = "Authentication::Private")] - pub async fn update( - data: actix_web::web::Data, + async fn update( + data: GuardedData, index_uid: actix_web::web::Path, body: actix_web::web::Json>, ) -> std::result::Result { @@ -49,9 +47,8 @@ macro_rules! make_setting_route { Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } - #[actix_web::get($route, wrap = "Authentication::Private")] - pub async fn get( - data: actix_web::web::Data, + async fn get( + data: GuardedData, index_uid: actix_web::web::Path, ) -> std::result::Result { let settings = data.settings(index_uid.into_inner()).await?; @@ -60,6 +57,13 @@ macro_rules! make_setting_route { let val = json[$camelcase_attr].take(); Ok(HttpResponse::Ok().json(val)) } + + pub fn resources() -> Resource { + Resource::new($route) + .route(web::get().to(get)) + .route(web::post().to(update)) + .route(web::delete().to(delete)) + } } }; } @@ -117,14 +121,11 @@ macro_rules! create_services { ($($mod:ident),*) => { pub fn services(cfg: &mut web::ServiceConfig) { cfg - .service(update_all) - .service(get_all) - .service(delete_all) - $( - .service($mod::get) - .service($mod::update) - .service($mod::delete) - )*; + .service(web::resource("/indexes/{index_uid}/settings") + .route(web::post().to(update_all)) + .route(web::get().to(get_all)) + .route(web::delete().to(delete_all))) + $(.service($mod::resources()))*; } }; } @@ -139,9 +140,8 @@ create_services!( ranking_rules ); -#[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] async fn update_all( - data: web::Data, + data: GuardedData, index_uid: web::Path, body: web::Json>, ) -> Result { @@ -154,9 +154,8 @@ async fn update_all( Ok(HttpResponse::Accepted().json(json)) } -#[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] async fn get_all( - data: web::Data, + data: GuardedData, index_uid: web::Path, ) -> Result { let settings = data.settings(index_uid.into_inner()).await?; @@ -164,9 +163,8 @@ async fn get_all( Ok(HttpResponse::Ok().json(settings)) } -#[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] async fn delete_all( - data: web::Data, + data: GuardedData, index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); From 561596d8bc7ec8d9bb08cee863cfa8c29c45503c Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:54:51 +0200 Subject: [PATCH 502/522] update stats routes --- meilisearch-http/src/routes/stats.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index c39989188..bfebba6ed 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,23 +1,20 @@ -use actix_web::get; -use actix_web::web; -use actix_web::HttpResponse; use log::debug; +use actix_web::{web, HttpResponse}; use serde::Serialize; use crate::error::ResponseError; -use crate::helpers::Authentication; +use crate::extractors::authentication::{GuardedData, policies::*}; use crate::routes::IndexParam; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get_index_stats) - .service(get_stats) - .service(get_version); + cfg.route("/indexes/{index_uid}/stats", web::get().to(get_index_stats)) + .route("/stats", web::get().to(get_stats)) + .route("/version", web::get().to(get_version)); } -#[get("/indexes/{index_uid}/stats", wrap = "Authentication::Private")] async fn get_index_stats( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { let response = data.get_index_stats(path.index_uid.clone()).await?; @@ -26,8 +23,7 @@ async fn get_index_stats( Ok(HttpResponse::Ok().json(response)) } -#[get("/stats", wrap = "Authentication::Private")] -async fn get_stats(data: web::Data) -> Result { +async fn get_stats(data: GuardedData) -> Result { let response = data.get_all_stats().await?; debug!("returns: {:?}", response); @@ -42,8 +38,7 @@ struct VersionResponse { pkg_version: String, } -#[get("/version", wrap = "Authentication::Private")] -async fn get_version() -> HttpResponse { +async fn get_version(_data: GuardedData) -> HttpResponse { let commit_sha = match option_env!("COMMIT_SHA") { Some("") | None => env!("VERGEN_SHA"), Some(commit_sha) => commit_sha, From d078cbf39b3036a397512e17bb832a6c75ac0460 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 15:56:12 +0200 Subject: [PATCH 503/522] remove authentication middleware --- .../src/helpers/authentication.rs | 150 ------------------ meilisearch-http/src/helpers/mod.rs | 2 - 2 files changed, 152 deletions(-) delete mode 100644 meilisearch-http/src/helpers/authentication.rs diff --git a/meilisearch-http/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs deleted file mode 100644 index dddf57138..000000000 --- a/meilisearch-http/src/helpers/authentication.rs +++ /dev/null @@ -1,150 +0,0 @@ -use std::pin::Pin; -use std::task::{Context, Poll}; - -use actix_web::body::Body; -use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; -use actix_web::web; -use actix_web::ResponseError as _; -use futures::future::{ok, Future, Ready}; -use futures::ready; -use pin_project::pin_project; - -use crate::error::{AuthenticationError, ResponseError}; -use crate::Data; - -#[derive(Clone, Copy)] -pub enum Authentication { - Public, - Private, - Admin, -} - -impl Transform for Authentication -where - S: Service, Error = actix_web::Error>, -{ - type Response = ServiceResponse; - type Error = actix_web::Error; - type InitError = (); - type Transform = LoggingMiddleware; - type Future = Ready>; - - fn new_transform(&self, service: S) -> Self::Future { - ok(LoggingMiddleware { - acl: *self, - service, - }) - } -} - -pub struct LoggingMiddleware { - acl: Authentication, - service: S, -} - -#[allow(clippy::type_complexity)] -impl Service for LoggingMiddleware -where - S: Service, Error = actix_web::Error>, -{ - type Response = ServiceResponse; - type Error = actix_web::Error; - type Future = AuthenticationFuture; - - fn poll_ready(&self, cx: &mut Context) -> Poll> { - self.service.poll_ready(cx) - } - - fn call(&self, req: ServiceRequest) -> Self::Future { - let data = req.app_data::>().unwrap(); - - if data.api_keys().master.is_none() { - return AuthenticationFuture::Authenticated(self.service.call(req)); - } - - let auth_header = match req.headers().get("X-Meili-API-Key") { - Some(auth) => match auth.to_str() { - Ok(auth) => auth, - Err(_) => return AuthenticationFuture::NoHeader(Some(req)), - }, - None => return AuthenticationFuture::NoHeader(Some(req)), - }; - - let authenticated = match self.acl { - Authentication::Admin => data.api_keys().master.as_deref() == Some(auth_header), - Authentication::Private => { - data.api_keys().master.as_deref() == Some(auth_header) - || data.api_keys().private.as_deref() == Some(auth_header) - } - Authentication::Public => { - data.api_keys().master.as_deref() == Some(auth_header) - || data.api_keys().private.as_deref() == Some(auth_header) - || data.api_keys().public.as_deref() == Some(auth_header) - } - }; - - if authenticated { - AuthenticationFuture::Authenticated(self.service.call(req)) - } else { - AuthenticationFuture::Refused(Some(req)) - } - } -} - -#[pin_project(project = AuthProj)] -pub enum AuthenticationFuture -where - S: Service, -{ - Authenticated(#[pin] S::Future), - NoHeader(Option), - Refused(Option), -} - -impl Future for AuthenticationFuture -where - S: Service, Error = actix_web::Error>, -{ - type Output = Result, actix_web::Error>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.project(); - match this { - AuthProj::Authenticated(fut) => match ready!(fut.poll(cx)) { - Ok(resp) => Poll::Ready(Ok(resp)), - Err(e) => Poll::Ready(Err(e)), - }, - AuthProj::NoHeader(req) => { - match req.take() { - Some(req) => { - let response = - ResponseError::from(AuthenticationError::MissingAuthorizationHeader); - let response = response.error_response(); - let response = req.into_response(response); - Poll::Ready(Ok(response)) - } - // https://doc.rust-lang.org/nightly/std/future/trait.Future.html#panics - None => unreachable!("poll called again on ready future"), - } - } - AuthProj::Refused(req) => { - match req.take() { - Some(req) => { - let bad_token = req - .headers() - .get("X-Meili-API-Key") - .map(|h| h.to_str().map(String::from).unwrap_or_default()) - .unwrap_or_default(); - let response = - ResponseError::from(AuthenticationError::InvalidToken(bad_token)); - let response = response.error_response(); - let response = req.into_response(response); - Poll::Ready(Ok(response)) - } - // https://doc.rust-lang.org/nightly/std/future/trait.Future.html#panics - None => unreachable!("poll called again on ready future"), - } - } - } - } -} diff --git a/meilisearch-http/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs index a5cddf29c..c664f15aa 100644 --- a/meilisearch-http/src/helpers/mod.rs +++ b/meilisearch-http/src/helpers/mod.rs @@ -1,6 +1,4 @@ -pub mod authentication; pub mod compression; mod env; -pub use authentication::Authentication; pub use env::EnvSizer; From 8e4928c7eaf25f7e174252ccf1acf99077bb5684 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 16:25:27 +0200 Subject: [PATCH 504/522] fix tests --- meilisearch-http/src/routes/document.rs | 15 ++++++++------- meilisearch-http/src/routes/health.rs | 2 +- .../tests/documents/delete_documents.rs | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 4824ec7b8..abea85ed6 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -44,14 +44,15 @@ pub fn services(cfg: &mut web::ServiceConfig) { .route(web::put().guard(guard_json).to(update_documents)) .route(web::delete().to(clear_all_documents)), ) + // this route needs to be before the /documents/{document_id} to match properly + .route( + "/indexes/{index_uid}/documents/delete-batch", + web::post().to(delete_documents), + ) .service( - web::scope("/indexes/{index_uid}/documents/") - .service( - web::resource("{document_id}") - .route(web::get().to(get_document)) - .route(web::delete().to(delete_document)), - ) - .route("/delete-batch", web::post().to(delete_documents)), + web::resource("/indexes/{index_uid}/documents/{document_id}") + .route(web::get().to(get_document)) + .route(web::delete().to(delete_document)), ); } diff --git a/meilisearch-http/src/routes/health.rs b/meilisearch-http/src/routes/health.rs index 172298861..3c7200200 100644 --- a/meilisearch-http/src/routes/health.rs +++ b/meilisearch-http/src/routes/health.rs @@ -3,7 +3,7 @@ use actix_web::{web, HttpResponse}; use crate::error::ResponseError; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.route("/healts", web::get().to(get_health)); + cfg.route("/health", web::get().to(get_health)); } async fn get_health() -> Result { diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs index d5794e40c..eb6fa040b 100644 --- a/meilisearch-http/tests/documents/delete_documents.rs +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -14,8 +14,8 @@ async fn delete_one_unexisting_document() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; - let (_response, code) = index.delete_document(0).await; - assert_eq!(code, 202); + let (response, code) = index.delete_document(0).await; + assert_eq!(code, 202, "{}", response); let update = index.wait_update_id(0).await; assert_eq!(update["status"], "processed"); } @@ -85,8 +85,8 @@ async fn clear_all_documents_empty_index() { #[actix_rt::test] async fn delete_batch_unexisting_index() { let server = Server::new().await; - let (_response, code) = server.index("test").delete_batch(vec![]).await; - assert_eq!(code, 404); + let (response, code) = server.index("test").delete_batch(vec![]).await; + assert_eq!(code, 404, "{}", response); } #[actix_rt::test] From 79fc3bb84e3659797fc3ca962854fc026e7b6684 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 16:25:52 +0200 Subject: [PATCH 505/522] fmt --- meilisearch-http/src/lib.rs | 40 ++++++++++--------- meilisearch-http/src/routes/key.rs | 2 +- meilisearch-http/src/routes/search.rs | 2 +- meilisearch-http/src/routes/settings.rs | 2 +- meilisearch-http/src/routes/stats.rs | 2 +- .../tests/settings/get_settings.rs | 7 +++- 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 78d9ec8b1..ffdf62eb7 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -19,8 +19,8 @@ pub use option::Opt; use actix_web::web; -use extractors::payload::PayloadConfig; use extractors::authentication::policies::*; +use extractors::payload::PayloadConfig; pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { let http_payload_size_limit = data.http_payload_size_limit(); @@ -42,7 +42,7 @@ pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { pub fn configure_auth(config: &mut web::ServiceConfig, data: &Data) { let keys = data.api_keys(); - let auth_config = if let Some(ref master_key) = keys.master { + let auth_config = if let Some(ref master_key) = keys.master { let private_key = keys.private.as_ref().unwrap(); let public_key = keys.public.as_ref().unwrap(); let mut policies = init_policies!(Public, Private, Admin); @@ -62,8 +62,8 @@ pub fn configure_auth(config: &mut web::ServiceConfig, data: &Data) { #[cfg(feature = "mini-dashboard")] pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { - use actix_web_static_files::Resource; use actix_web::HttpResponse; + use actix_web_static_files::Resource; mod generated { include!(concat!(env!("OUT_DIR"), "/generated.rs")); @@ -71,22 +71,24 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { if enable_frontend { let generated = generated::generate(); - let mut scope = web::scope("/"); - // Generate routes for mini-dashboard assets - for (path, resource) in generated.into_iter() { - let Resource {mime_type, data, ..} = resource; - // Redirect index.html to / - if path == "index.html" { - config.service(web::resource("/").route(web::get().to(move || { - HttpResponse::Ok().content_type(mime_type).body(data) - }))); - } else { - scope = scope.service(web::resource(path).route(web::get().to(move || { - HttpResponse::Ok().content_type(mime_type).body(data) - }))); - } + let mut scope = web::scope("/"); + // Generate routes for mini-dashboard assets + for (path, resource) in generated.into_iter() { + let Resource { + mime_type, data, .. + } = resource; + // Redirect index.html to / + if path == "index.html" { + config.service(web::resource("/").route( + web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)), + )); + } else { + scope = scope.service(web::resource(path).route( + web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)), + )); } - config.service(scope); + } + config.service(scope); } else { config.route("/", web::get().to(routes::running)); } @@ -105,7 +107,7 @@ macro_rules! create_app { use actix_web::App; use actix_web::{middleware, web}; use meilisearch_http::routes::*; - use meilisearch_http::{configure_data, dashboard, configure_auth}; + use meilisearch_http::{configure_auth, configure_data, dashboard}; App::new() .configure(|s| configure_data(s, $data.clone())) diff --git a/meilisearch-http/src/routes/key.rs b/meilisearch-http/src/routes/key.rs index 51b1c78a2..1ea400a4d 100644 --- a/meilisearch-http/src/routes/key.rs +++ b/meilisearch-http/src/routes/key.rs @@ -1,8 +1,8 @@ use actix_web::{web, HttpResponse}; use serde::Serialize; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; -use crate::extractors::authentication::{GuardedData, policies::*}; pub fn services(cfg: &mut web::ServiceConfig) { cfg.route("/keys", web::get().to(list)); diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 7f1e265f6..7307a5990 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -6,10 +6,10 @@ use serde::Deserialize; use serde_json::Value; use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; -use crate::extractors::authentication::{GuardedData, policies::*}; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service( diff --git a/meilisearch-http/src/routes/settings.rs b/meilisearch-http/src/routes/settings.rs index f7df7b701..6a8d9cca5 100644 --- a/meilisearch-http/src/routes/settings.rs +++ b/meilisearch-http/src/routes/settings.rs @@ -1,7 +1,7 @@ use log::debug; use actix_web::{web, HttpResponse}; -use crate::extractors::authentication::{GuardedData, policies::*}; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::index::Settings; use crate::Data; use crate::{error::ResponseError, index::Unchecked}; diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index bfebba6ed..33bb482d9 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -3,7 +3,7 @@ use actix_web::{web, HttpResponse}; use serde::Serialize; use crate::error::ResponseError; -use crate::extractors::authentication::{GuardedData, policies::*}; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::routes::IndexParam; use crate::Data; diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 4941b5d2f..0b523eef3 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use once_cell::sync::Lazy; -use serde_json::{Value, json}; +use serde_json::{json, Value}; use crate::common::Server; @@ -11,7 +11,10 @@ static DEFAULT_SETTINGS_VALUES: Lazy> = Lazy::new(| map.insert("searchable_attributes", json!(["*"])); map.insert("filterable_attributes", json!([])); map.insert("distinct_attribute", json!(Value::Null)); - map.insert("ranking_rules", json!(["words", "typo", "proximity", "attribute", "exactness"])); + map.insert( + "ranking_rules", + json!(["words", "typo", "proximity", "attribute", "exactness"]), + ); map.insert("stop_words", json!([])); map.insert("synonyms", json!({})); map From fbd58f2eec0e93e7052633c9105a940cd8f5a61e Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 16:27:13 +0200 Subject: [PATCH 506/522] clippy --- meilisearch-http/src/extractors/authentication/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index 2cce6911d..fd3272e2f 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -16,7 +16,7 @@ macro_rules! create_policies { use crate::extractors::authentication::Policy; $( - #[derive(Debug)] + #[derive(Debug, Default)] pub struct $name { inner: HashSet> } From b1f7fe24f6a497b4a71716559186faeea580dcb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 24 Jun 2021 16:45:51 +0200 Subject: [PATCH 507/522] Fix docker build --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 8c2648512..81ddf4a53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ FROM alpine:3.10 AS compiler RUN apk update --quiet RUN apk add curl RUN apk add build-base +RUN apk add libressl-dev RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From 3b601f615a455a8c089c86dc06cc92f5de3f4782 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 16:53:20 +0200 Subject: [PATCH 508/522] declare new authentication related errors --- meilisearch-http/src/error.rs | 17 ------------ .../src/extractors/authentication/error.rs | 26 +++++++++++++++++++ .../src/extractors/authentication/mod.rs | 13 ++++++---- 3 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 meilisearch-http/src/extractors/authentication/error.rs diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 6124ed880..4f47abd66 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -10,23 +10,6 @@ use meilisearch_error::{Code, ErrorCode}; use milli::UserError; use serde::{Deserialize, Serialize}; -#[derive(Debug, thiserror::Error)] -pub enum AuthenticationError { - #[error("You must have an authorization token")] - MissingAuthorizationHeader, - #[error("Invalid API key")] - InvalidToken(String), -} - -impl ErrorCode for AuthenticationError { - fn error_code(&self) -> Code { - match self { - AuthenticationError::MissingAuthorizationHeader => Code::MissingAuthorizationHeader, - AuthenticationError::InvalidToken(_) => Code::InvalidToken, - } - } -} - #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct ResponseError { diff --git a/meilisearch-http/src/extractors/authentication/error.rs b/meilisearch-http/src/extractors/authentication/error.rs new file mode 100644 index 000000000..29578e373 --- /dev/null +++ b/meilisearch-http/src/extractors/authentication/error.rs @@ -0,0 +1,26 @@ +use meilisearch_error::{Code, ErrorCode}; + +#[derive(Debug, thiserror::Error)] +pub enum AuthenticationError { + #[error("You must have an authorization token")] + MissingAuthorizationHeader, + #[error("Invalid API key")] + InvalidToken(String), + // Triggered on configuration error. + #[error("Irretrievable state")] + IrretrievableState, + #[error("Unknown authentication policy")] + UnknownPolicy, +} + +impl ErrorCode for AuthenticationError { + fn error_code(&self) -> Code { + match self { + AuthenticationError::MissingAuthorizationHeader => Code::MissingAuthorizationHeader, + AuthenticationError::InvalidToken(_) => Code::InvalidToken, + AuthenticationError::IrretrievableState => Code::Internal, + AuthenticationError::UnknownPolicy => Code::Internal, + } + } +} + diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index fd3272e2f..6b9ac24ae 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -1,3 +1,5 @@ +mod error; + use std::any::{Any, TypeId}; use std::collections::HashMap; use std::marker::PhantomData; @@ -7,7 +9,8 @@ use actix_web::FromRequest; use futures::future::err; use futures::future::{ok, Ready}; -use crate::error::{AuthenticationError, ResponseError}; +use crate::error::ResponseError; +use error::AuthenticationError; macro_rules! create_policies { ($($name:ident), *) => { @@ -151,7 +154,7 @@ impl FromRequest for GuardedData data, _marker: PhantomData, }), - None => todo!("Data not configured"), + None => err(AuthenticationError::IrretrievableState.into()), }, AuthConfig::Auth(policies) => match policies.get::

() { Some(policy) => match req.headers().get("x-meili-api-key") { @@ -162,7 +165,7 @@ impl FromRequest for GuardedData data, _marker: PhantomData, }), - None => todo!("Data not configured"), + None => err(AuthenticationError::IrretrievableState.into()), } } else { err(AuthenticationError::InvalidToken(String::from("hello")).into()) @@ -170,10 +173,10 @@ impl FromRequest for GuardedData } None => err(AuthenticationError::MissingAuthorizationHeader.into()), }, - None => todo!("no policy found"), + None => err(AuthenticationError::UnknownPolicy.into()), }, }, - None => todo!(), + None => err(AuthenticationError::IrretrievableState.into()), } } } From 01b09c065b29da38ce78e25b75bf1becc9e58df8 Mon Sep 17 00:00:00 2001 From: marin postma Date: Thu, 24 Jun 2021 19:02:28 +0200 Subject: [PATCH 509/522] change route to service --- meilisearch-http/src/lib.rs | 4 ++-- meilisearch-http/src/routes/document.rs | 30 ++++++++++++------------- meilisearch-http/src/routes/dump.rs | 4 ++-- meilisearch-http/src/routes/health.rs | 2 +- meilisearch-http/src/routes/index.rs | 12 +++++----- meilisearch-http/src/routes/key.rs | 2 +- meilisearch-http/src/routes/stats.rs | 6 ++--- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index ffdf62eb7..0eb61f84c 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -90,13 +90,13 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { } config.service(scope); } else { - config.route("/", web::get().to(routes::running)); + config.service(web::resource("/").route(web::get().to(routes::running))); } } #[cfg(not(feature = "mini-dashboard"))] pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { - config.route("/", web::get().to(routes::running)); + config.service(web::resource("/").route(web::get().to(routes::running))); } #[macro_export] diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index abea85ed6..817f624d0 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -38,21 +38,21 @@ struct DocumentParam { pub fn services(cfg: &mut web::ServiceConfig) { cfg.service( - web::resource("/indexes/{index_uid}/documents") - .route(web::get().to(get_all_documents)) - .route(web::post().guard(guard_json).to(add_documents)) - .route(web::put().guard(guard_json).to(update_documents)) - .route(web::delete().to(clear_all_documents)), - ) - // this route needs to be before the /documents/{document_id} to match properly - .route( - "/indexes/{index_uid}/documents/delete-batch", - web::post().to(delete_documents), - ) - .service( - web::resource("/indexes/{index_uid}/documents/{document_id}") - .route(web::get().to(get_document)) - .route(web::delete().to(delete_document)), + web::scope("/indexes/{index_uid}/documents") + .service( + web::resource("") + .route(web::get().to(get_all_documents)) + .route(web::post().guard(guard_json).to(add_documents)) + .route(web::put().guard(guard_json).to(update_documents)) + .route(web::delete().to(clear_all_documents)), + ) + // this route needs to be before the /documents/{document_id} to match properly + .service(web::resource("/delete-batch").route(web::post().to(delete_documents))) + .service( + web::resource("/{document_id}") + .route(web::get().to(get_document)) + .route(web::delete().to(delete_document)), + ), ); } diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index f2f276332..e506755a1 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -7,8 +7,8 @@ use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.route("/dumps", web::post().to(create_dump)) - .route("/dumps/{dump_uid}/status", web::get().to(get_dump_status)); + cfg.service(web::resource("/dumps").route(web::post().to(create_dump))) + .service(web::resource("/dumps/{dump_uid}/status").route(web::get().to(get_dump_status))); } async fn create_dump(data: GuardedData) -> Result { diff --git a/meilisearch-http/src/routes/health.rs b/meilisearch-http/src/routes/health.rs index 3c7200200..54237de1a 100644 --- a/meilisearch-http/src/routes/health.rs +++ b/meilisearch-http/src/routes/health.rs @@ -3,7 +3,7 @@ use actix_web::{web, HttpResponse}; use crate::error::ResponseError; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.route("/health", web::get().to(get_health)); + cfg.service(web::resource("/health").route(web::get().to(get_health))); } async fn get_health() -> Result { diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 76a6d5da8..eb8da92ed 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -20,13 +20,13 @@ pub fn services(cfg: &mut web::ServiceConfig) { .route(web::put().to(update_index)) .route(web::delete().to(delete_index)), ) - .route( - "/indexes/{index_uid}/updates", - web::get().to(get_all_updates_status), + .service( + web::resource("/indexes/{index_uid}/updates") + .route(web::get().to(get_all_updates_status)) ) - .route( - "/indexes/{index_uid}/updates/{update_id}", - web::get().to(get_update_status), + .service( + web::resource("/indexes/{index_uid}/updates/{update_id}") + .route(web::get().to(get_update_status)) ); } diff --git a/meilisearch-http/src/routes/key.rs b/meilisearch-http/src/routes/key.rs index 1ea400a4d..d47e264be 100644 --- a/meilisearch-http/src/routes/key.rs +++ b/meilisearch-http/src/routes/key.rs @@ -5,7 +5,7 @@ use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.route("/keys", web::get().to(list)); + cfg.service(web::resource("/keys").route(web::get().to(list))); } #[derive(Serialize)] diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index 33bb482d9..e440ce8ff 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -8,9 +8,9 @@ use crate::routes::IndexParam; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.route("/indexes/{index_uid}/stats", web::get().to(get_index_stats)) - .route("/stats", web::get().to(get_stats)) - .route("/version", web::get().to(get_version)); + cfg.service(web::resource("/indexes/{index_uid}/stats").route(web::get().to(get_index_stats))) + .service(web::resource("/stats").route(web::get().to(get_stats))) + .service(web::resource("/version").route(web::get().to(get_version))); } async fn get_index_stats( From c1c50f6714fc60d69ee5c83ce00e203fca05efe7 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 28 Jun 2021 13:35:25 +0200 Subject: [PATCH 510/522] unused borrow that must be used --- meilisearch-http/src/extractors/authentication/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index 6b9ac24ae..13ced2248 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -30,7 +30,7 @@ macro_rules! create_policies { } pub fn add(&mut self, token: Vec) { - &mut self.inner.insert(token); + self.inner.insert(token); } } From d74c698adc17ffc65fa04767d450a2b24df625a1 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 28 Jun 2021 13:23:13 +0200 Subject: [PATCH 511/522] stop logging the no space left on device error --- meilisearch-http/src/main.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 3c51b7e72..5638c453f 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -7,6 +7,8 @@ use structopt::StructOpt; #[cfg(all(not(debug_assertions), feature = "analytics"))] use meilisearch_http::analytics; +#[cfg(all(not(debug_assertions), feature = "analytics"))] +use std::sync::Arc; #[cfg(target_os = "linux")] #[global_allocator] @@ -21,7 +23,7 @@ async fn main() -> Result<(), MainError> { let mut log_builder = env_logger::Builder::new(); log_builder.parse_filters(&opt.log_level); - if opt.log_level == "info" { + if opt.log_level == "info" { // if we are in info we only allow the warn log_level for milli log_builder.filter_module("milli", log::LevelFilter::Warn); } @@ -46,6 +48,14 @@ async fn main() -> Result<(), MainError> { let sentry = sentry::init(sentry::ClientOptions { release: sentry::release_name!(), dsn: Some(SENTRY_DSN.parse()?), + before_send: Some(Arc::new(|event| { + event + .message + .as_ref() + .map(|msg| msg.to_lowercase().contains("no space left on device")) + .unwrap_or(false) + .then(|| event) + })), ..Default::default() }); // sentry must stay alive as long as possible From a59f437ee39432e9c056523e07404efcc961d1fa Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 28 Jun 2021 14:35:50 +0200 Subject: [PATCH 512/522] use only half of the computer threads for the indexation by default --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 1 + meilisearch-http/src/index/update_handler.rs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1eee0fe65..de2ffc0dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1631,6 +1631,7 @@ dependencies = [ "milli", "mime", "mockall", + "num_cpus", "obkv", "once_cell", "oxidized-json-checker", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 5d2fdbcad..353401382 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -53,6 +53,7 @@ meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.6.0" } mime = "0.3.16" +num_cpus = "1.13.0" once_cell = "1.5.2" oxidized-json-checker = "0.3.2" parking_lot = "0.11.1" diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs index 9896c9391..4d860ed7e 100644 --- a/meilisearch-http/src/index/update_handler.rs +++ b/meilisearch-http/src/index/update_handler.rs @@ -23,7 +23,7 @@ pub struct UpdateHandler { impl UpdateHandler { pub fn new(opt: &IndexerOpts) -> anyhow::Result { let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(opt.indexing_jobs.unwrap_or(0)) + .num_threads(opt.indexing_jobs.unwrap_or(num_cpus::get() / 2)) .build()?; Ok(Self { max_nb_chunks: opt.max_nb_chunks, From a26bb50d62e2c79f1adb9dbc64cf23ac713da184 Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Mon, 28 Jun 2021 15:13:52 +0200 Subject: [PATCH 513/522] Update mini-dashboard to v.0.1.3 --- meilisearch-http/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 5d2fdbcad..20a3792d5 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -118,5 +118,5 @@ default = ["analytics", "mini-dashboard"] jemallocator = "0.3.2" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.2/build.zip" -sha1 = "881949457a710a22b5cefcb0418038ceb8b0eb26" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.3/build.zip" +sha1 = "fea1780e13d8e570e35a1921e7a45cabcd501d5e" From 348d1123882b1c3433d43b179a30d1cf1457fda9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 28 Jun 2021 16:55:29 +0200 Subject: [PATCH 514/522] Fix docker run --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 81ddf4a53..3f55c9ae7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN $HOME/.cargo/bin/cargo build --release # Run FROM alpine:3.10 -RUN apk add -q --no-cache libgcc tini +RUN apk add -q --no-cache libgcc tini libressl-dev COPY --from=compiler /meilisearch/target/release/meilisearch . From c09e610bb5e921ff03b2b469c6691ba9be2d231f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 29 Jun 2021 10:25:47 +0200 Subject: [PATCH 515/522] Update heed and milli --- Cargo.lock | 20 ++++++++----------- meilisearch-http/Cargo.toml | 4 ++-- .../update_actor/store/mod.rs | 8 ++++++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de2ffc0dc..9ad1dff8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1176,9 +1176,8 @@ dependencies = [ [[package]] name = "heed" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" +version = "0.12.0" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" dependencies = [ "byteorder", "heed-traits", @@ -1196,14 +1195,12 @@ dependencies = [ [[package]] name = "heed-traits" version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" [[package]] name = "heed-types" version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" dependencies = [ "bincode", "heed-traits", @@ -1492,9 +1489,8 @@ checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lmdb-rkv-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b27470ac25167b3afdfb6af8fcd3bc1be67de50ffbdaf4073378cfded6ae24a5" +version = "0.15.0" +source = "git+https://github.com/meilisearch/lmdb-rs#d0b50d02938ee84e4e4372697ea991fe2a4cae3b" dependencies = [ "cc", "libc", @@ -1724,8 +1720,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.6.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.6.0#c38b0b883d602a7efe9ba87f36157f4fc35f6ec7" +version = "0.7.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.0#9dbc8b2dd06e12f0ee551d877261223fa4968110" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 4d4d4f90d..8cbea77b7 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -42,7 +42,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = "0.10.6" +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } itertools = "0.10.0" @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.6.0" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.0" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 21a320b26..cf5b846c6 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -439,7 +439,9 @@ impl UpdateStore { while let Some(Ok(((_, uuid, _), pending))) = pendings.next() { if uuid == index_uuid { - pendings.del_current()?; + unsafe { + pendings.del_current()?; + } let mut pending = pending.decode()?; if let Some(update_uuid) = pending.content.take() { uuids_to_remove.push(update_uuid); @@ -456,7 +458,9 @@ impl UpdateStore { .lazily_decode_data(); while let Some(_) = updates.next() { - updates.del_current()?; + unsafe { + updates.del_current()?; + } } drop(updates); From 1dc99ea4515d257148f9aec53cfc78a8bd5b90bb Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 29 Jun 2021 11:57:47 +0200 Subject: [PATCH 516/522] accept no content-type as json --- meilisearch-http/src/routes/document.rs | 16 ++++ .../tests/documents/add_documents.rs | 78 ++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 817f624d0..eb41f7eff 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -13,6 +13,7 @@ use crate::Data; const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; +/* macro_rules! guard_content_type { ($fn_name:ident, $guard_value:literal) => { fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { @@ -29,6 +30,19 @@ macro_rules! guard_content_type { } guard_content_type!(guard_json, "application/json"); +*/ + +fn guard_json(head: &actix_web::dev::RequestHead) -> bool { + if let Some(content_type) = head.headers.get("Content-Type") { + content_type + .to_str() + .map(|v| v.contains("application/json")) + // if no content-type is specified we still accept the data as json! + .unwrap_or(true) + } else { + false + } + } #[derive(Deserialize)] struct DocumentParam { @@ -146,6 +160,8 @@ async fn add_documents( Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } +/// Route used when the payload type is "application/json" +/// Used to add or replace documents async fn update_documents( data: GuardedData, path: web::Path, diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 5d2a91674..66e475172 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -1,7 +1,83 @@ +use crate::common::{GetAllDocumentsOptions, Server}; +use actix_web::test; use chrono::DateTime; +use meilisearch_http::create_app; use serde_json::{json, Value}; -use crate::common::{GetAllDocumentsOptions, Server}; +/// This is the basic usage of our API and every other tests uses the content-type application/json +#[actix_rt::test] +async fn add_documents_test_json_content_types() { + let document = json!([ + { + "id": 1, + "content": "Bouvier Bernois", + } + ]); + + // this is a what is expected and should work + let server = Server::new().await; + let app = test::init_service(create_app!(&server.service.0, true)).await; + let req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(document.to_string()) + .insert_header(("content-type", "application/json")) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response, json!({ "updateId": 0 })); +} + +/// no content type is still supposed to be accepted as json +#[actix_rt::test] +async fn add_documents_test_no_content_types() { + let document = json!([ + { + "id": 1, + "content": "Montagne des Pyrénées", + } + ]); + + let server = Server::new().await; + let app = test::init_service(create_app!(&server.service.0, true)).await; + let req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(document.to_string()) + .insert_header(("content-type", "application/json")) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response, json!({ "updateId": 0 })); +} + +/// any other content-type is must be refused +#[actix_rt::test] +async fn add_documents_test_bad_content_types() { + let document = json!([ + { + "id": 1, + "content": "Leonberg", + } + ]); + + let server = Server::new().await; + let app = test::init_service(create_app!(&server.service.0, true)).await; + let req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(document.to_string()) + .insert_header(("content-type", "text/plain")) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + assert_eq!(status_code, 405); + assert!(body.is_empty()); +} #[actix_rt::test] async fn add_documents_no_index_creation() { From ec809ca48718a329ca510278d6eaaca815ae4bd9 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 29 Jun 2021 13:07:40 +0200 Subject: [PATCH 517/522] use rustls instead of openssl and remove all default-features of reqwest --- Cargo.lock | 147 ------------------------------------ meilisearch-http/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ad1dff8b..66bca3d12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -643,22 +643,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "core-foundation" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" - [[package]] name = "cow-utils" version = "0.1.2" @@ -905,21 +889,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1326,19 +1295,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes 1.0.1", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "idna" version = "0.2.3" @@ -1834,24 +1790,6 @@ dependencies = [ "syn 1.0.73", ] -[[package]] -name = "native-tls" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nix" version = "0.19.1" @@ -1941,39 +1879,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "openssl" -version = "0.10.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" -dependencies = [ - "bitflags", - "cfg-if 1.0.0", - "foreign-types", - "libc", - "once_cell", - "openssl-sys", -] - -[[package]] -name = "openssl-probe" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" - -[[package]] -name = "openssl-sys" -version = "0.9.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "ordered-float" version = "2.5.1" @@ -2494,13 +2399,11 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", - "hyper-tls", "ipnet", "js-sys", "lazy_static", "log", "mime", - "native-tls", "percent-encoding", "pin-project-lite", "rustls", @@ -2508,7 +2411,6 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", - "tokio-native-tls", "tokio-rustls", "url", "wasm-bindgen", @@ -2611,16 +2513,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" -dependencies = [ - "lazy_static", - "winapi", -] - [[package]] name = "scopeguard" version = "1.1.0" @@ -2637,29 +2529,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "security-framework" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "semver" version = "0.9.0" @@ -3261,16 +3130,6 @@ dependencies = [ "syn 1.0.73", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.22.0" @@ -3456,12 +3315,6 @@ dependencies = [ "serde", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "vec_map" version = "0.8.2" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 8cbea77b7..ccd6e9055 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -76,7 +76,7 @@ walkdir = "2.3.2" obkv = "0.1.1" pin-project = "1.0.7" whoami = { version = "1.1.2", optional = true } -reqwest = { version = "0.11.3", features = ["json"], optional = true } +reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true } [dependencies.sentry] default-features = false From fe7640555d5675a1a0298da8cd5b9b1c8f8c70ac Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 29 Jun 2021 13:16:56 +0200 Subject: [PATCH 518/522] fix the content-type --- meilisearch-http/src/routes/document.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index eb41f7eff..5f0d1d1fa 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -37,10 +37,10 @@ fn guard_json(head: &actix_web::dev::RequestHead) -> bool { content_type .to_str() .map(|v| v.contains("application/json")) - // if no content-type is specified we still accept the data as json! - .unwrap_or(true) + .unwrap_or(false) } else { - false + // if no content-type is specified we still accept the data as json! + true } } From c28246675005e05614b39dfd522e283d8b525d65 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 29 Jun 2021 15:22:11 +0200 Subject: [PATCH 519/522] remove the libressl dependency from our docker file --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f55c9ae7..8c2648512 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ FROM alpine:3.10 AS compiler RUN apk update --quiet RUN apk add curl RUN apk add build-base -RUN apk add libressl-dev RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -35,7 +34,7 @@ RUN $HOME/.cargo/bin/cargo build --release # Run FROM alpine:3.10 -RUN apk add -q --no-cache libgcc tini libressl-dev +RUN apk add -q --no-cache libgcc tini COPY --from=compiler /meilisearch/target/release/meilisearch . From 29bf6a8d4250ff2f7944e831147718a6c0e5b533 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 29 Jun 2021 15:25:18 +0200 Subject: [PATCH 520/522] run rustfmt one the whole project and add it to the CI --- .github/workflows/rust.yml | 14 +++++++++++++ .../src/extractors/authentication/error.rs | 1 - .../index_controller/update_actor/actor.rs | 2 +- meilisearch-http/src/routes/document.rs | 20 +++++++++---------- meilisearch-http/src/routes/dump.rs | 2 +- meilisearch-http/src/routes/index.rs | 5 ++--- meilisearch-http/src/routes/search.rs | 2 +- meilisearch-http/src/routes/settings.rs | 2 +- meilisearch-http/src/routes/stats.rs | 2 +- 9 files changed, 31 insertions(+), 19 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8020ead32..5a8403d6d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -65,3 +65,17 @@ jobs: with: command: clippy args: --all-targets -- --deny warnings + + fmt: + name: Run Rustfmt + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + components: rustfmt + - name: Run cargo fmt + run: cargo fmt --all -- --check diff --git a/meilisearch-http/src/extractors/authentication/error.rs b/meilisearch-http/src/extractors/authentication/error.rs index 29578e373..902634045 100644 --- a/meilisearch-http/src/extractors/authentication/error.rs +++ b/meilisearch-http/src/extractors/authentication/error.rs @@ -23,4 +23,3 @@ impl ErrorCode for AuthenticationError { } } } - diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 51e8cb28c..8ba96dad1 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_stream::stream; use futures::StreamExt; -use log::{trace}; +use log::trace; use oxidized_json_checker::JsonChecker; use tokio::fs; use tokio::io::AsyncWriteExt; diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 5f0d1d1fa..418c67462 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -33,16 +33,16 @@ guard_content_type!(guard_json, "application/json"); */ fn guard_json(head: &actix_web::dev::RequestHead) -> bool { - if let Some(content_type) = head.headers.get("Content-Type") { - content_type - .to_str() - .map(|v| v.contains("application/json")) - .unwrap_or(false) - } else { - // if no content-type is specified we still accept the data as json! - true - } - } + if let Some(content_type) = head.headers.get("Content-Type") { + content_type + .to_str() + .map(|v| v.contains("application/json")) + .unwrap_or(false) + } else { + // if no content-type is specified we still accept the data as json! + true + } +} #[derive(Deserialize)] struct DocumentParam { diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index e506755a1..1f987a588 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -1,5 +1,5 @@ -use log::debug; use actix_web::{web, HttpResponse}; +use log::debug; use serde::{Deserialize, Serialize}; use crate::error::ResponseError; diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index eb8da92ed..badbdcc10 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -21,12 +21,11 @@ pub fn services(cfg: &mut web::ServiceConfig) { .route(web::delete().to(delete_index)), ) .service( - web::resource("/indexes/{index_uid}/updates") - .route(web::get().to(get_all_updates_status)) + web::resource("/indexes/{index_uid}/updates").route(web::get().to(get_all_updates_status)), ) .service( web::resource("/indexes/{index_uid}/updates/{update_id}") - .route(web::get().to(get_update_status)) + .route(web::get().to(get_update_status)), ); } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 7307a5990..31a7dbd03 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeSet, HashSet}; -use log::debug; use actix_web::{web, HttpResponse}; +use log::debug; use serde::Deserialize; use serde_json::Value; diff --git a/meilisearch-http/src/routes/settings.rs b/meilisearch-http/src/routes/settings.rs index 6a8d9cca5..812e37b58 100644 --- a/meilisearch-http/src/routes/settings.rs +++ b/meilisearch-http/src/routes/settings.rs @@ -1,5 +1,5 @@ -use log::debug; use actix_web::{web, HttpResponse}; +use log::debug; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::index::Settings; diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index e440ce8ff..a0078d76a 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,5 +1,5 @@ -use log::debug; use actix_web::{web, HttpResponse}; +use log::debug; use serde::Serialize; use crate::error::ResponseError; From 3011209e2852f53ed46904d0b2d7240862bede24 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 29 Jun 2021 16:36:41 +0200 Subject: [PATCH 521/522] bump alpine version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8c2648512..811be5c14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Compile -FROM alpine:3.10 AS compiler +FROM alpine:3.14 AS compiler RUN apk update --quiet RUN apk add curl @@ -32,7 +32,7 @@ COPY . . RUN $HOME/.cargo/bin/cargo build --release # Run -FROM alpine:3.10 +FROM alpine:3.14 RUN apk add -q --no-cache libgcc tini From 3a9b86ad559d347b5125dc1bbc43b227e4a6071d Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 30 Jun 2021 10:49:10 +0200 Subject: [PATCH 522/522] add rustfmt to bors --- bors.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bors.toml b/bors.toml index 45133f836..e3348c36d 100644 --- a/bors.toml +++ b/bors.toml @@ -2,7 +2,8 @@ status = [ 'Tests on ubuntu-18.04', 'Tests on macos-latest', 'Cargo check on Windows', - 'Run Clippy' + 'Run Clippy', + 'Run Rustfmt' ] # 3 hours timeout timeout-sec = 10800